You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2015/03/23 10:15:13 UTC

[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2819 - Still Failing

Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2819/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:25463/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:25463/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([10BC4AC8B8E2CBDE:98E87512161EA626]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([10BC4AC8B8E2CBDE:98E87512161EA626]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([10BC4AC8B8E2CBDE:98E87512161EA626]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9274 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/init-core-data-001
   [junit4]   2> 104037 T324 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /y/x
   [junit4]   2> 104044 T324 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 104045 T325 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 104145 T324 oasc.ZkTestServer.run start zk server on port:12131
   [junit4]   2> 104146 T324 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 104147 T324 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 104151 T332 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@495d114a name:ZooKeeperConnection Watcher:127.0.0.1:12131 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 104151 T324 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 104152 T324 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 104152 T324 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 104155 T324 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 104156 T324 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 104157 T335 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@317cc8c6 name:ZooKeeperConnection Watcher:127.0.0.1:12131/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 104158 T324 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 104158 T324 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 104159 T324 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 104161 T324 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 104164 T324 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 104165 T324 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 104168 T324 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 104169 T324 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 104172 T324 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 104173 T324 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 104176 T324 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 104177 T324 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 104180 T324 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 104181 T324 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 104183 T324 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 104184 T324 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 104186 T324 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 104187 T324 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 104190 T324 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 104190 T324 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 104193 T324 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 104193 T324 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 104196 T324 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 104197 T324 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 104199 T324 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 104200 T324 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 104202 T324 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 104203 T324 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 106597 T324 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/control-001/cores/collection1
   [junit4]   2> 106602 T324 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 106605 T324 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:12144
   [junit4]   2> 106606 T324 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/control-001/cores, hostPort=12143, hostContext=/y/x}
   [junit4]   2> 106607 T324 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 106607 T324 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/control-001/'
   [junit4]   2> 106674 T324 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/control-001/solr.xml
   [junit4]   2> 106754 T324 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/control-001/cores
   [junit4]   2> 106755 T324 oasc.CoreContainer.<init> New CoreContainer 262157179
   [junit4]   2> 106755 T324 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/control-001/]
   [junit4]   2> 106756 T324 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/control-001/lib
   [junit4]   2> 106757 T324 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/control-001/lib).
   [junit4]   2> 106771 T324 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 106771 T324 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 106771 T324 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 106772 T324 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 106772 T324 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 106773 T324 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 106773 T324 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 106774 T324 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 106774 T324 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 106774 T324 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 106775 T324 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 106776 T324 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 106776 T324 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 106777 T324 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 106777 T324 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 106778 T324 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:12131/solr
   [junit4]   2> 106778 T324 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 106779 T324 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 106780 T324 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 106782 T347 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6d8e0c3b name:ZooKeeperConnection Watcher:127.0.0.1:12131 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 106782 T324 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 106783 T324 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 106786 T324 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 106787 T350 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1031fb8e name:ZooKeeperConnection Watcher:127.0.0.1:12131/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 106787 T324 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 106789 T324 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 106792 T324 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 106794 T324 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 106797 T324 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 106799 T324 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 106802 T324 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 106804 T324 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 106806 T324 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 106807 T324 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:12143_y%2Fx
   [junit4]   2> 106808 T324 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:12143_y%2Fx
   [junit4]   2> 106810 T324 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 106812 T324 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 106813 T324 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 106815 T324 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:12143_y%2Fx
   [junit4]   2> 106816 T324 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 106818 T324 oasc.Overseer.start Overseer (id=93526337601077251-127.0.0.1:12143_y%2Fx-n_0000000000) starting
   [junit4]   2> 106820 T324 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 106827 T324 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 106828 T352 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 106828 T324 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 106828 T351 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 106833 T324 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/control-001/cores
   [junit4]   2> 106836 T324 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, collection=control_collection, coreNodeName=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/control-001/cores/collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/control-001/cores/collection1/, shard=, schema=schema.xml, transient=false, loadOnStartup=true, name=collection1, dataDir=data/}
   [junit4]   2> 106837 T324 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/control-001/cores/collection1/
   [junit4]   2> 106837 T324 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 106838 T354 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 106838 T354 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 106840 T350 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 106840 T354 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 106841 T351 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:12143_y%2Fx",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:12143/y/x",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null} current state version: 0
   [junit4]   2> 106842 T351 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:12143_y%2Fx",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:12143/y/x",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 106842 T351 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 106842 T351 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 106844 T350 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 107840 T354 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 107841 T354 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 107842 T354 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 107842 T354 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 107843 T354 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 107843 T354 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/control-001/cores/collection1/'
   [junit4]   2> 107857 T354 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 107859 T354 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 107871 T354 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 107890 T354 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 107932 T354 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 107934 T354 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 107949 T354 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 108447 T354 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 108473 T354 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 108477 T354 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 108507 T354 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 108513 T354 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 108519 T354 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 108520 T354 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 108521 T354 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 108521 T354 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 108522 T354 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 108522 T354 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 108523 T354 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 108523 T354 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 108524 T354 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/control-001/cores/collection1/data/
   [junit4]   2> 108524 T354 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@41e2b5e6
   [junit4]   2> 108525 T354 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/control-001/cores/collection1/data
   [junit4]   2> 108526 T354 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/control-001/cores/collection1/data/index/
   [junit4]   2> 108526 T354 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 108527 T354 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/control-001/cores/collection1/data/index
   [junit4]   2> 108527 T354 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=25, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 108529 T354 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@53c49f42 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@11b3d105),segFN=segments_1,generation=1}
   [junit4]   2> 108529 T354 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 108554 T354 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 108555 T354 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 108555 T354 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 108555 T354 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 108556 T354 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 108556 T354 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 108557 T354 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 108558 T354 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 108558 T354 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 108591 T354 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 108595 T354 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 108598 T354 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 108601 T354 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 108609 T354 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update,/replication,/schema,/update/csv,/get,/admin/segments,/admin/file,/config,/update/json/docs,/admin/logging,/admin/plugins,/admin/mbeans,/admin/luke,standard,/admin/system,/admin/properties,/admin/ping,/admin/threads,/update/json
   [junit4]   2> 108611 T354 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 108614 T354 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 108614 T354 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 108616 T354 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 108617 T354 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 108618 T354 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=13, maxMergeAtOnceExplicit=14, maxMergedSegmentMB=2.6337890625, floorSegmentMB=1.427734375, forceMergeDeletesPctAllowed=21.115639387685633, segmentsPerTier=30.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6094095464987471
   [junit4]   2> 108619 T354 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@53c49f42 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@11b3d105),segFN=segments_1,generation=1}
   [junit4]   2> 108619 T354 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 108620 T354 oass.SolrIndexSearcher.<init> Opening Searcher@3dd4482a[collection1] main
   [junit4]   2> 108620 T354 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 108621 T354 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 108622 T354 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 108622 T354 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 108623 T354 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 108623 T354 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 108624 T354 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 108625 T354 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 108625 T354 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 108625 T354 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 108627 T355 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3dd4482a[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 108627 T354 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 108628 T358 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:12143/y/x collection:control_collection shard:shard1
   [junit4]   2> 108629 T324 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 108629 T324 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 108629 T358 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 108634 T324 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 108635 T324 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 108636 T358 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 108636 T362 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3d67fcee name:ZooKeeperConnection Watcher:127.0.0.1:12131/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 108637 T324 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 108638 T324 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 108638 T350 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 108638 T358 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 108638 T324 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 108639 T358 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 108639 T351 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C9 name=collection1 org.apache.solr.core.SolrCore@10c58a51 url=http://127.0.0.1:12143/y/x/collection1 node=127.0.0.1:12143_y%2Fx C9_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:12143_y%2Fx, state=down, core=collection1, base_url=http://127.0.0.1:12143/y/x}
   [junit4]   2> 108640 T358 C9 P12143 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:12143/y/x/collection1/
   [junit4]   2> 108640 T358 C9 P12143 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 108641 T358 C9 P12143 oasc.SyncStrategy.syncToMe http://127.0.0.1:12143/y/x/collection1/ has no replicas
   [junit4]   2> 108641 T358 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:12143/y/x/collection1/ shard1
   [junit4]   2> 108642 T358 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 108642 T350 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 108642 T362 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 108644 T324 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 108644 T324 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 108645 T324 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 108646 T324 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 108647 T365 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7d4ecb90 name:ZooKeeperConnection Watcher:127.0.0.1:12131/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 108648 T324 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 108648 T350 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 108648 T324 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 108650 T351 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:12143/y/x",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 108654 T351 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 108654 T351 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 108655 T350 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 108655 T362 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 108659 T350 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 108660 T351 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 108662 T350 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 108662 T362 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 108700 T358 oasc.ZkController.register We are http://127.0.0.1:12143/y/x/collection1/ and leader is http://127.0.0.1:12143/y/x/collection1/
   [junit4]   2> 108701 T358 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:12143/y/x
   [junit4]   2> 108701 T358 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 108701 T358 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 108701 T358 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 108703 T350 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 108704 T351 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:12143_y%2Fx",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:12143/y/x",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null} current state version: 4
   [junit4]   2> 108705 T351 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:12143_y%2Fx",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:12143/y/x",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 108811 T350 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 108811 T362 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 110395 T324 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/shard-1-001/cores/collection1
   [junit4]   2> 110399 T324 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/shard-1-001
   [junit4]   2> 110399 T324 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 110402 T324 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:12163
   [junit4]   2> 110402 T324 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/y/x, hostPort=12162, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/shard-1-001/cores, solrconfig=solrconfig.xml, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/tempDir-001/jetty1}
   [junit4]   2> 110403 T324 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 110403 T324 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/shard-1-001/'
   [junit4]   2> 110447 T324 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/shard-1-001/solr.xml
   [junit4]   2> 110502 T324 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/shard-1-001/cores
   [junit4]   2> 110503 T324 oasc.CoreContainer.<init> New CoreContainer 802522050
   [junit4]   2> 110503 T324 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/shard-1-001/]
   [junit4]   2> 110503 T324 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/shard-1-001/lib
   [junit4]   2> 110504 T324 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/shard-1-001/lib).
   [junit4]   2> 110514 T324 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 110515 T324 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 110515 T324 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 110515 T324 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 110516 T324 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 110516 T324 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 110516 T324 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 110516 T324 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 110517 T324 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 110517 T324 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 110517 T324 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 110518 T324 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 110518 T324 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 110519 T324 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 110519 T324 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 110519 T324 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:12131/solr
   [junit4]   2> 110520 T324 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 110520 T324 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 110521 T324 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 110522 T377 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3caa3588 name:ZooKeeperConnection Watcher:127.0.0.1:12131 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 110523 T324 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 110524 T324 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 110526 T324 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 110527 T380 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@65b1e8f4 name:ZooKeeperConnection Watcher:127.0.0.1:12131/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 110527 T324 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 110532 T324 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 111538 T324 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:12162_y%2Fx
   [junit4]   2> 111539 T324 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:12162_y%2Fx
   [junit4]   2> 111542 T324 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 111545 T324 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/shard-1-001/cores
   [junit4]   2> 111547 T324 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {schema=schema.xml, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/shard-1-001/cores/collection1, name=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/shard-1-001/cores/collection1/, shard=, coreNodeName=, loadOnStartup=true, dataDir=data/, collection=collection1, transient=false}
   [junit4]   2> 111548 T324 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/shard-1-001/cores/collection1/
   [junit4]   2> 111548 T324 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 111550 T381 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 111550 T381 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 111552 T350 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 111553 T381 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 111553 T381 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 111553 T351 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:12162_y%2Fx",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:12162/y/x",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null} current state version: 5
   [junit4]   2> 111553 T381 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 111554 T351 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:12162_y%2Fx",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:12162/y/x",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 111554 T351 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 111554 T351 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 111554 T381 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 111555 T381 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 111556 T351 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 111556 T380 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 111557 T380 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 112555 T381 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 112556 T381 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 112557 T381 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 112557 T381 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 112558 T381 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 112558 T381 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/shard-1-001/cores/collection1/'
   [junit4]   2> 112571 T381 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 112574 T381 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 112584 T381 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 112603 T381 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 112647 T381 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 112648 T381 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 112662 T381 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 112989 T381 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 113003 T381 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 113005 T381 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 113024 T381 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 113028 T381 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 113032 T381 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 113033 T381 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 113033 T381 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 113033 T381 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 113034 T381 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 113034 T381 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 113035 T381 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 113035 T381 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 113035 T381 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 113036 T381 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@41e2b5e6
   [junit4]   2> 113037 T381 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/shard-1-001/cores/collection1/data
   [junit4]   2> 113037 T381 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 113037 T381 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 113038 T381 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 113038 T381 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=25, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 113039 T381 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@53370ce8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@43307e06),segFN=segments_1,generation=1}
   [junit4]   2> 113039 T381 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 113046 T381 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 113046 T381 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 113047 T381 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 113047 T381 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 113047 T381 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 113047 T381 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 113048 T381 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 113048 T381 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 113048 T381 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 113063 T381 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 113065 T381 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 113066 T381 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 113068 T381 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 113076 T381 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update,/replication,/schema,/update/csv,/get,/admin/segments,/admin/file,/config,/update/json/docs,/admin/logging,/admin/plugins,/admin/mbeans,/admin/luke,standard,/admin/system,/admin/properties,/admin/ping,/admin/threads,/update/json
   [junit4]   2> 113078 T381 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 113080 T381 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 113080 T381 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 113081 T381 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 113081 T381 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 113082 T381 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=13, maxMergeAtOnceExplicit=14, maxMergedSegmentMB=2.6337890625, floorSegmentMB=1.427734375, forceMergeDeletesPctAllowed=21.115639387685633, segmentsPerTier=30.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6094095464987471
   [junit4]   2> 113083 T381 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@53370ce8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@43307e06),segFN=segments_1,generation=1}
   [junit4]   2> 113083 T381 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 113083 T381 oass.SolrIndexSearcher.<init> Opening Searcher@8f8348e[collection1] main
   [junit4]   2> 113084 T381 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 113085 T381 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 113085 T381 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 113086 T381 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 113086 T381 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 113086 T381 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 113087 T381 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 113087 T381 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 113087 T381 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 113087 T381 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 113089 T382 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@8f8348e[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 113089 T381 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 113090 T385 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:12162/y/x collection:collection1 shard:shard1
   [junit4]   2> 113090 T324 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 113091 T324 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 113091 T385 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 113095 T385 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 113097 T350 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 113097 T385 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 113097 T385 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C10 name=collection1 org.apache.solr.core.SolrCore@3a21d734 url=http://127.0.0.1:12162/y/x/collection1 node=127.0.0.1:12162_y%2Fx C10_STATE=coll:collection1 core:collection1 props:{node_name=127.0.0.1:12162_y%2Fx, state=down, core=collection1, base_url=http://127.0.0.1:12162/y/x}
   [junit4]   2> 113097 T385 C10 P12162 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:12162/y/x/collection1/
   [junit4]   2> 113098 T385 C10 P12162 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 113098 T351 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 113098 T385 C10 P12162 oasc.SyncStrategy.syncToMe http://127.0.0.1:12162/y/x/collection1/ has no replicas
   [junit4]   2> 113098 T385 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:12162/y/x/collection1/ shard1
   [junit4]   2> 113099 T385 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 113099 T351 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 113100 T380 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 113101 T380 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 113103 T351 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:12162/y/x",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 113104 T351 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 113105 T380 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 113106 T380 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 113107 T350 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 113153 T385 oasc.ZkController.register We are http://127.0.0.1:12162/y/x/collection1/ and leader is http://127.0.0.1:12162/y/x/collection1/
   [junit4]   2> 113154 T385 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:12162/y/x
   [junit4]   2> 113154 T385 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 113154 T385 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 113154 T385 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 113156 T350 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 113157 T351 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:12162_y%2Fx",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:12162/y/x",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null} current state version: 5
   [junit4]   2> 113158 T351 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:12162_y%2Fx",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:12162/y/x",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 113159 T351 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 3
   [junit4]   2> 113159 T385 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 113160 T380 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 113161 T380 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 4 
   [junit4]   2> 114957 T324 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/shard-2-001/cores/collection1
   [junit4]   2> 114960 T324 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/shard-2-001
   [junit4]   2> 114961 T324 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 114963 T324 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:12177
   [junit4]   2> 114963 T324 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/y/x, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/tempDir-001/jetty2, hostPort=12176, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/shard-2-001/cores, solrconfig=solrconfig.xml}
   [junit4]   2> 114964 T324 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 114964 T324 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/shard-2-001/'
   [junit4]   2> 114997 T324 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/shard-2-001/solr.xml
   [junit4]   2> 115040 T324 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/shard-2-001/cores
   [junit4]   2> 115040 T324 oasc.CoreContainer.<init> New CoreContainer 589785909
   [junit4]   2> 115041 T324 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/shard-2-001/]
   [junit4]   2> 115041 T324 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/shard-2-001/lib
   [junit4]   2> 115042 T324 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 10BC4AC8B8E2CBDE-001/shard-2-001/lib).
   [junit4]   2> 115051 T324 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 115051 T324 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 115052 T324 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 115052 T324 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 115052 T324 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 115052 T324 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 115053 T324 oashc.HttpShardHandlerFactory.getParame

[...truncated too long message...]

it4]   2> 		5	/solr/aliases.json
   [junit4]   2> 		5	/solr/clusterstate.json
   [junit4]   2> 		4	/solr/configs/conf1
   [junit4]   2> 		3	/solr/collections/repfacttest_c8n_1x3/state.json
   [junit4]   2> 		3	/solr/collections/collection1/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		5	/solr/live_nodes
   [junit4]   2> 		4	/solr/overseer/collection-queue-work
   [junit4]   2> 		4	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1451154 T4929 oasc.SocketProxy.close WARN Closing 1 connections to: http://127.0.0.1:27222/
   [junit4]   2> 1451154 T4929 oasc.SocketProxy.close WARN Closing 3 connections to: http://127.0.0.1:27251/
   [junit4]   2> 1451155 T4929 oasc.SocketProxy.close WARN Closing 10 connections to: http://127.0.0.1:27242/
   [junit4]   2> 1451155 T4929 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:27235/
   [junit4]   2> 1451155 T4929 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 1451156 T4929 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:27222/
   [junit4]   2> 1451156 T4929 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:27251/
   [junit4]   2> 1451156 T4929 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:27242/
   [junit4]   2> 1451156 T4929 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:27235/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=10BC4AC8B8E2CBDE -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=en_SG -Dtests.timezone=Asia/Gaza -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   73.3s J1 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([10BC4AC8B8E2CBDE:98E87512161EA626]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 1451178 T4929 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 10BC4AC8B8E2CBDE-001
   [junit4]   2> 73310 T4928 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {id=PostingsFormat(name=Direct), range_facet_l=FST50, text=PostingsFormat(name=MockRandom), _version_=FST50, multiDefault=Lucene50(blocksize=128), a_t=FST50, timestamp=FST50, intDefault=FST50}, docValues:{timestamp=DocValuesFormat(name=Direct)}, sim=DefaultSimilarity, locale=en_SG, timezone=Asia/Gaza
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=192443576,total=362807296
   [junit4]   2> NOTE: All tests run in this JVM: [SharedFSAutoReplicaFailoverUtilsTest, BadComponentTest, TestPostingsSolrHighlighter, TestSurroundQueryParser, TestFunctionQuery, HdfsDirectoryTest, SaslZkACLProviderTest, TestStressReorder, OverseerStatusTest, AnalyticsMergeStrategyTest, StatsComponentTest, TestQueryUtils, CachingDirectoryFactoryTest, TestDocumentBuilder, TestExpandComponent, TestMiniSolrCloudClusterSSL, TestIBSimilarityFactory, TestFileDictionaryLookup, TestStressRecovery, TestRandomMergePolicy, TestInfoStreamLogging, TestFieldTypeResource, NoCacheHeaderTest, SolrXmlInZkTest, DistributedIntervalFacetingTest, HdfsRecoveryZkTest, TestStressLucene, ZkSolrClientTest, OverseerRolesTest, ExitableDirectoryReaderTest, ZkControllerTest, LeaderInitiatedRecoveryOnCommitTest, TestCSVResponseWriter, DocValuesTest, CollectionsAPIDistributedZkTest, RequiredFieldsTest, DistributedTermsComponentTest, BadCopyFieldTest, SimpleMLTQParserTest, SuggesterWFSTTest, DistributedQueryComponentCustomSortTest, SpatialHeatmapFacetsTest, BinaryUpdateRequestHandlerTest, TestFastOutputStream, TestSerializedLuceneMatchVersion, SolrCoreCheckLockOnStartupTest, StandardRequestHandlerTest, DirectUpdateHandlerTest, TestRebalanceLeaders, TestMiniSolrCloudCluster, TestCSVLoader, FastVectorHighlighterTest, RequestHandlersTest, SolrCmdDistributorTest, ZkNodePropsTest, HdfsLockFactoryTest, LeaderFailoverAfterPartitionTest, ChaosMonkeyNothingIsSafeTest, CustomCollectionTest, ReplicationFactorTest]
   [junit4] Completed on J1 in 73.59s, 1 test, 1 error <<< FAILURES!

[...truncated 699 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 475 suites, 1892 tests, 3 errors, 62 ignored (22 assumptions)

Total time: 51 minutes 17 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 203277265 bytes
Compression is 0.0%
Took 39 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2918 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2918/

1 tests failed.
REGRESSION:  org.apache.solr.cloud.RecoveryZkTest.test

Error Message:
shard1 is not consistent.  Got 963 from http://127.0.0.1:32047/esw/zr/collection1lastClient and got 272 from http://127.0.0.1:32051/esw/zr/collection1

Stack Trace:
java.lang.AssertionError: shard1 is not consistent.  Got 963 from http://127.0.0.1:32047/esw/zr/collection1lastClient and got 272 from http://127.0.0.1:32051/esw/zr/collection1
	at __randomizedtesting.SeedInfo.seed([CED94C4AA2172441:468D73900CEB49B9]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.RecoveryZkTest.test(RecoveryZkTest.java:123)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 9725 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryZkTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/init-core-data-001
   [junit4]   2> 648134 T2610 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 648134 T2610 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /esw/zr
   [junit4]   2> 648141 T2610 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 648141 T2611 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 648241 T2610 oasc.ZkTestServer.run start zk server on port:32033
   [junit4]   2> 648242 T2610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 648243 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 648247 T2618 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3d31f642 name:ZooKeeperConnection Watcher:127.0.0.1:32033 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 648247 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 648248 T2610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 648248 T2610 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 648251 T2610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 648252 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 648254 T2621 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@50c588 name:ZooKeeperConnection Watcher:127.0.0.1:32033/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 648254 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 648255 T2610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 648255 T2610 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 648257 T2610 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 648259 T2610 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 648260 T2610 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 648261 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 648262 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 648265 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 648266 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 648268 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 648268 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 648270 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 648271 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 648273 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 648273 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 648275 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 648276 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 648277 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 648278 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 648280 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 648281 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 648282 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 648283 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 648285 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 648285 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 648287 T2610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 648288 T2610 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 649512 T2610 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores/collection1
   [junit4]   2> 649515 T2610 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 649517 T2610 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:32039
   [junit4]   2> 649518 T2610 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=32039, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores, hostContext=/esw/zr}
   [junit4]   2> 649518 T2610 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 649518 T2610 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/'
   [junit4]   2> 649548 T2610 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/solr.xml
   [junit4]   2> 649583 T2610 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores
   [junit4]   2> 649583 T2610 oasc.CoreContainer.<init> New CoreContainer 235084862
   [junit4]   2> 649584 T2610 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/]
   [junit4]   2> 649584 T2610 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/lib
   [junit4]   2> 649584 T2610 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/lib).
   [junit4]   2> 649594 T2610 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 649594 T2610 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 649594 T2610 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 649595 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 649595 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 649595 T2610 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 649595 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 649595 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 649596 T2610 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 649596 T2610 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 649596 T2610 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 649601 T2610 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 649602 T2610 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 649602 T2610 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 649602 T2610 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 649602 T2610 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:32033/solr
   [junit4]   2> 649603 T2610 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 649603 T2610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 649603 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 649605 T2633 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@64a4870 name:ZooKeeperConnection Watcher:127.0.0.1:32033 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 649606 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 649606 T2610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 649608 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 649609 T2636 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@74862f9b name:ZooKeeperConnection Watcher:127.0.0.1:32033/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 649609 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 649611 T2610 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 649612 T2610 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 649613 T2610 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 649615 T2610 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 649616 T2610 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 649617 T2610 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 649618 T2610 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 649619 T2610 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 649620 T2610 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:32039_esw%2Fzr
   [junit4]   2> 649621 T2610 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:32039_esw%2Fzr
   [junit4]   2> 649622 T2610 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 649623 T2610 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 649624 T2610 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 649625 T2610 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:32039_esw%2Fzr
   [junit4]   2> 649625 T2610 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 649626 T2610 oasc.Overseer.start Overseer (id=93624945334288387-127.0.0.1:32039_esw%2Fzr-n_0000000000) starting
   [junit4]   2> 649627 T2610 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 649631 T2610 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 649632 T2638 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 649632 T2637 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 649632 T2610 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 649635 T2610 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores
   [junit4]   2> 649636 T2610 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {transient=false, collection=control_collection, coreNodeName=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores/collection1/, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores/collection1, shard=, config=solrconfig.xml, dataDir=data/, loadOnStartup=true, name=collection1}
   [junit4]   2> 649637 T2610 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores/collection1/
   [junit4]   2> 649637 T2610 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 649637 T2640 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 649638 T2640 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 649638 T2636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 649638 T2640 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 649639 T2637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32039/esw/zr",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:32039_esw%2Fzr"} current state version: 0
   [junit4]   2> 649639 T2637 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32039/esw/zr",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:32039_esw%2Fzr"}
   [junit4]   2> 649639 T2637 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 649640 T2637 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 649641 T2636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 650639 T2640 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 650640 T2640 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 650640 T2640 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 650641 T2640 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 650641 T2640 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 650641 T2640 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores/collection1/'
   [junit4]   2> 650659 T2640 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 650661 T2640 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 650671 T2640 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 650692 T2640 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 650740 T2640 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 650741 T2640 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 650750 T2640 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 651106 T2640 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 651109 T2640 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 651111 T2640 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 651116 T2640 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 651136 T2640 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 651136 T2640 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 651137 T2640 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores/collection1/data/
   [junit4]   2> 651137 T2640 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@74cee5ed
   [junit4]   2> 651138 T2640 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores/collection1/data
   [junit4]   2> 651138 T2640 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores/collection1/data/index/
   [junit4]   2> 651138 T2640 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 651139 T2640 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/control-001/cores/collection1/data/index
   [junit4]   2> 651139 T2640 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=10, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 651140 T2640 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@569f34ac lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6e3b34a1),segFN=segments_1,generation=1}
   [junit4]   2> 651141 T2640 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 651146 T2640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 651147 T2640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 651147 T2640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 651147 T2640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 651148 T2640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 651148 T2640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 651148 T2640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 651149 T2640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 651149 T2640 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 651171 T2640 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 651173 T2640 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 651175 T2640 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 651178 T2640 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 651183 T2640 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/segments,/schema,/admin/properties,/update/csv,/admin/file,/admin/mbeans,/config,/admin/luke,/admin/plugins,/admin/ping,/admin/threads,/update/json,/update/json/docs,/update,/get,/admin/system,/replication,standard,/admin/logging
   [junit4]   2> 651185 T2640 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 651187 T2640 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 651187 T2640 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 651189 T2640 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 651189 T2640 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 651190 T2640 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=46, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.15818545914049942]
   [junit4]   2> 651190 T2640 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@569f34ac lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6e3b34a1),segFN=segments_1,generation=1}
   [junit4]   2> 651191 T2640 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 651191 T2640 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@6f7491f8[collection1] main
   [junit4]   2> 651191 T2640 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 651192 T2640 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 651192 T2640 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 651193 T2640 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 651193 T2640 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 651193 T2640 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 651194 T2640 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 651194 T2640 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 651194 T2640 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 651195 T2640 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 651195 T2641 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6f7491f8[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 651196 T2640 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 651197 T2644 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:32039/esw/zr collection:control_collection shard:shard1
   [junit4]   2> 651197 T2610 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 651197 T2610 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 651198 T2644 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 651200 T2610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 651200 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 651202 T2644 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 651202 T2647 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6a411a6a name:ZooKeeperConnection Watcher:127.0.0.1:32033/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 651202 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 651203 T2610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 651203 T2636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 651203 T2644 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 651204 T2644 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 651203 T2610 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 651204 T2637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C1808 name=collection1 org.apache.solr.core.SolrCore@43fe35d1 url=http://127.0.0.1:32039/esw/zr/collection1 node=127.0.0.1:32039_esw%2Fzr C1808_STATE=coll:control_collection core:collection1 props:{core=collection1, state=down, base_url=http://127.0.0.1:32039/esw/zr, node_name=127.0.0.1:32039_esw%2Fzr}
   [junit4]   2> 651204 T2644 C:control_collection S:shard1 C1808 P32039 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:32039/esw/zr/collection1/
   [junit4]   2> 651206 T2644 C:control_collection S:shard1 C1808 P32039 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 651206 T2644 C:control_collection S:shard1 C1808 P32039 oasc.SyncStrategy.syncToMe http://127.0.0.1:32039/esw/zr/collection1/ has no replicas
   [junit4]   2> 651206 T2644 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:32039/esw/zr/collection1/ shard1
   [junit4]   2> 651207 T2644 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 651207 T2636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 651207 T2647 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 651208 T2610 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 651208 T2610 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 651209 T2610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 651209 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 651211 T2650 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@762874b2 name:ZooKeeperConnection Watcher:127.0.0.1:32033/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 651211 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 651212 T2636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 651212 T2610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 651213 T2637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32039/esw/zr",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 651215 T2637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 651216 T2637 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 651216 T2636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 651216 T2647 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 651219 T2636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 651220 T2637 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 651221 T2647 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 651221 T2636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 651263 T2644 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:32039/esw/zr/collection1/ and leader is http://127.0.0.1:32039/esw/zr/collection1/
   [junit4]   2> 651264 T2644 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:32039/esw/zr
   [junit4]   2> 651264 T2644 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 651264 T2644 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 651264 T2644 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 651265 T2636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 651266 T2637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32039/esw/zr",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:32039_esw%2Fzr"} current state version: 4
   [junit4]   2> 651267 T2637 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32039/esw/zr",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:32039_esw%2Fzr"}
   [junit4]   2> 651370 T2636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 651370 T2647 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 652802 T2610 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores/collection1
   [junit4]   2> 652804 T2610 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001
   [junit4]   2> 652805 T2610 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 652808 T2610 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:32047
   [junit4]   2> 652808 T2610 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/tempDir-001/jetty1, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores, hostContext=/esw/zr, hostPort=32047, solrconfig=solrconfig.xml}
   [junit4]   2> 652808 T2610 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 652808 T2610 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/'
   [junit4]   2> 652838 T2610 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/solr.xml
   [junit4]   2> 652872 T2610 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores
   [junit4]   2> 652873 T2610 oasc.CoreContainer.<init> New CoreContainer 1177738588
   [junit4]   2> 652873 T2610 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/]
   [junit4]   2> 652873 T2610 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/lib
   [junit4]   2> 652874 T2610 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/lib).
   [junit4]   2> 652883 T2610 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 652883 T2610 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 652883 T2610 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 652884 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 652884 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 652884 T2610 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 652884 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 652884 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 652885 T2610 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 652885 T2610 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 652885 T2610 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 652889 T2610 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 652890 T2610 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 652890 T2610 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 652891 T2610 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 652891 T2610 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:32033/solr
   [junit4]   2> 652891 T2610 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 652891 T2610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 652892 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 652894 T2662 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2dff8c1a name:ZooKeeperConnection Watcher:127.0.0.1:32033 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 652894 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 652895 T2610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 652896 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 652897 T2665 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4028f3af name:ZooKeeperConnection Watcher:127.0.0.1:32033/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 652897 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 652901 T2610 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 653904 T2610 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:32047_esw%2Fzr
   [junit4]   2> 653905 T2610 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:32047_esw%2Fzr
   [junit4]   2> 653909 T2610 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 653911 T2610 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores
   [junit4]   2> 653915 T2610 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores/collection1/, dataDir=data/, loadOnStartup=true, coreNodeName=, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores/collection1, collection=collection1, shard=, schema=schema.xml, transient=false, name=collection1}
   [junit4]   2> 653916 T2610 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores/collection1/
   [junit4]   2> 653916 T2610 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 653918 T2666 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 653918 T2666 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 653919 T2636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 653920 T2666 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 653920 T2637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32047/esw/zr",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:32047_esw%2Fzr"} current state version: 5
   [junit4]   2> 653921 T2666 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 653922 T2637 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32047/esw/zr",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:32047_esw%2Fzr"}
   [junit4]   2> 653922 T2666 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 653922 T2637 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 653922 T2637 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 653923 T2666 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 653923 T2666 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 653923 T2637 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 653924 T2665 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 653924 T2665 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 654923 T2666 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 654924 T2666 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 654924 T2666 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 654925 T2666 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 654925 T2666 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 654926 T2666 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores/collection1/'
   [junit4]   2> 654944 T2666 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 654947 T2666 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 654958 T2666 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 654982 T2666 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 655050 T2666 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 655052 T2666 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 655062 T2666 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 655423 T2666 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 655426 T2666 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 655428 T2666 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 655432 T2666 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 655451 T2666 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 655452 T2666 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 655452 T2666 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 655453 T2666 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@74cee5ed
   [junit4]   2> 655454 T2666 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores/collection1/data
   [junit4]   2> 655454 T2666 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 655454 T2666 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 655455 T2666 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 655455 T2666 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=10, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 655456 T2666 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5c4528fd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4086295e),segFN=segments_1,generation=1}
   [junit4]   2> 655456 T2666 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 655463 T2666 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 655464 T2666 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 655464 T2666 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 655464 T2666 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 655465 T2666 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 655465 T2666 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 655465 T2666 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 655466 T2666 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 655466 T2666 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 655486 T2666 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 655488 T2666 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 655491 T2666 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 655493 T2666 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 655502 T2666 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/segments,/schema,/admin/properties,/update/csv,/admin/file,/admin/mbeans,/config,/admin/luke,/admin/plugins,/admin/ping,/admin/threads,/update/json,/update/json/docs,/update,/get,/admin/system,/replication,standard,/admin/logging
   [junit4]   2> 655504 T2666 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 655506 T2666 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 655506 T2666 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 655508 T2666 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 655508 T2666 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 655509 T2666 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=46, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.15818545914049942]
   [junit4]   2> 655510 T2666 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5c4528fd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4086295e),segFN=segments_1,generation=1}
   [junit4]   2> 655510 T2666 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 655511 T2666 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@1e20960f[collection1] main
   [junit4]   2> 655511 T2666 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 655512 T2666 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 655512 T2666 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 655513 T2666 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 655513 T2666 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 655513 T2666 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 655513 T2666 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 655514 T2666 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 655514 T2666 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 655514 T2666 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 655515 T2667 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1e20960f[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 655516 T2666 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 655517 T2670 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:32047/esw/zr collection:collection1 shard:shard1
   [junit4]   2> 655517 T2610 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 655517 T2610 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 655517 T2670 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 655521 T2670 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 655522 T2636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 655522 T2670 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 655522 T2670 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C1809 name=collection1 org.apache.solr.core.SolrCore@1f1f534 url=http://127.0.0.1:32047/esw/zr/collection1 node=127.0.0.1:32047_esw%2Fzr C1809_STATE=coll:collection1 core:collection1 props:{core=collection1, state=down, base_url=http://127.0.0.1:32047/esw/zr, node_name=127.0.0.1:32047_esw%2Fzr}
   [junit4]   2> 655523 T2670 C:collection1 S:shard1 C1809 P32047 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:32047/esw/zr/collection1/
   [junit4]   2> 655523 T2637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 655523 T2670 C:collection1 S:shard1 C1809 P32047 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 655523 T2670 C:collection1 S:shard1 C1809 P32047 oasc.SyncStrategy.syncToMe http://127.0.0.1:32047/esw/zr/collection1/ has no replicas
   [junit4]   2> 655524 T2670 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:32047/esw/zr/collection1/ shard1
   [junit4]   2> 655524 T2670 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 655524 T2637 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 655525 T2665 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 655525 T2665 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 655527 T2637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32047/esw/zr",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 655528 T2637 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 655528 T2665 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 655529 T2665 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 655530 T2636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 655577 T2670 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:32047/esw/zr/collection1/ and leader is http://127.0.0.1:32047/esw/zr/collection1/
   [junit4]   2> 655578 T2670 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:32047/esw/zr
   [junit4]   2> 655578 T2670 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 655578 T2670 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 655578 T2670 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 655579 T2636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 655580 T2637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32047/esw/zr",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:32047_esw%2Fzr"} current state version: 5
   [junit4]   2> 655580 T2637 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32047/esw/zr",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:32047_esw%2Fzr"}
   [junit4]   2> 655581 T2670 C:collection1 S:shard1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 655581 T2637 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 3
   [junit4]   2> 655582 T2665 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 655582 T2665 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 4 
   [junit4]   2> 656725 T2610 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores/collection1
   [junit4]   2> 656728 T2610 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001
   [junit4]   2> 656728 T2610 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 656731 T2610 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:32051
   [junit4]   2> 656731 T2610 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/esw/zr, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/tempDir-001/jetty2, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores, hostPort=32051, solrconfig=solrconfig.xml}
   [junit4]   2> 656732 T2610 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 656732 T2610 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/'
   [junit4]   2> 656762 T2610 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/solr.xml
   [junit4]   2> 656797 T2610 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores
   [junit4]   2> 656797 T2610 oasc.CoreContainer.<init> New CoreContainer 192842349
   [junit4]   2> 656797 T2610 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/]
   [junit4]   2> 656798 T2610 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/lib
   [junit4]   2> 656798 T2610 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/lib).
   [junit4]   2> 656807 T2610 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 656808 T2610 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 656808 T2610 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 656808 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 656808 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 656808 T2610 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 656809 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 656809 T2610 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 656809 T2610 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 656809 T2610 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 656809 T2610 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 656814 T2610 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 656815 T2610 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 656815 T2610 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 656815 T2610 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 656815 T2610 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:32033/solr
   [junit4]   2> 656816 T2610 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 656816 T2610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 656817 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 656818 T2682 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@60b18fb0 name:ZooKeeperConnection Watcher:127.0.0.1:32033 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 656819 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 656819 T2610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 656821 T2610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 656822 T2685 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@21576f30 name:ZooKeeperConnection Watcher:127.0.0.1:32033/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 656822 T2610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 656825 T2610 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 657829 T2610 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:32051_esw%2Fzr
   [junit4]   2> 657830 T2610 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:32051_esw%2Fzr
   [junit4]   2> 657833 T2610 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 657835 T2610 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores
   [junit4]   2> 657838 T2610 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, dataDir=data/, shard=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores/collection1/, loadOnStartup=true, collection=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores/collection1, name=collection1, transient=false, schema=schema.xml, coreNodeName=}
   [junit4]   2> 657838 T2610 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores/collection1/
   [junit4]   2> 657838 T2610 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 657840 T2686 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 657840 T2686 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 657841 T2636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 657842 T2686 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 657842 T2686 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 657842 T2637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32051/esw/zr",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:32051_esw%2Fzr"} current state version: 5
   [junit4]   2> 657843 T2686 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 657843 T2637 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32051/esw/zr",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:32051_esw%2Fzr"}
   [junit4]   2> 657843 T2637 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 657843 T2637 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 657843 T2686 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 4 
   [junit4]   2> 657844 T2686 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 657844 T2637 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 4
   [junit4]   2> 657845 T2685 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 3)
   [junit4]   2> 657845 T2665 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 3)
   [junit4]   2> 657845 T2685 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 5 
   [junit4]   2> 657845 T2665 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 5 
   [junit4]   2> 658844 T2686 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 658845 T2686 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 658845 T2686 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 658846 T2686 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 658846 T2686 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 658846 T2686 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores/collection1/'
   [junit4]   2> 658865 T2686 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 658867 T2686 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 658877 T2686 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 658899 T2686 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 658948 T2686 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 658949 T2686 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 658959 T2686 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 659306 T2686 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 659309 T2686 C:collection

[...truncated too long message...]

ds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=492,cumulative_deletesById=226,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=56680,transaction_logs_total_number=1}
   [junit4]   2> 819687 T2729 C:collection1 S:shard1 c:collection1 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 819687 T2729 C:collection1 S:shard1 c:collection1 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 819687 T2729 C:collection1 S:shard1 c:collection1 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 819689 T2729 C:collection1 S:shard1 c:collection1 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 819697 T2729 C:collection1 S:shard1 c:collection1 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 819697 T2729 C:collection1 S:shard1 c:collection1 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores/collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores/collection1/data;done=false>>]
   [junit4]   2> 819697 T2729 C:collection1 S:shard1 c:collection1 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores/collection1/data
   [junit4]   2> 819698 T2729 C:collection1 S:shard1 c:collection1 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores/collection1/data/index [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores/collection1/data/index;done=false>>]
   [junit4]   2> 819698 T2729 C:collection1 S:shard1 c:collection1 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001/shard-2-001/cores/collection1/data/index
   [junit4]   2> 820389 T2736 C:collection1 S:shard1 c:collection1 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 820391 T2610 C:collection1 S:shard1 c:collection1 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/esw/zr,null}
   [junit4]   2> 820594 T2610 C:collection1 S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:32033 32033
   [junit4]   2> 820765 T2611 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:32033 32033
   [junit4]   2> 820767 T2611 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		5	/solr/aliases.json
   [junit4]   2> 		5	/solr/clusterstate.json
   [junit4]   2> 		4	/solr/configs/conf1
   [junit4]   2> 		3	/solr/collections/collection1/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent data watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		2	/solr/overseer_elect/election/93624945334288391-127.0.0.1:32047_esw%2Fzr-n_0000000001
   [junit4]   2> 		2	/solr/collections/collection1/leader_elect/shard1/election/93624945334288391-core_node1-n_0000000000
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryZkTest -Dtests.method=test -Dtests.seed=CED94C4AA2172441 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ja -Dtests.timezone=Canada/Saskatchewan -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE  173s J3 | RecoveryZkTest.test <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: shard1 is not consistent.  Got 963 from http://127.0.0.1:32047/esw/zr/collection1lastClient and got 272 from http://127.0.0.1:32051/esw/zr/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([CED94C4AA2172441:468D73900CEB49B9]:0)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryZkTest.test(RecoveryZkTest.java:123)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 820776 T2610 C:collection1 S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryZkTest CED94C4AA2172441-001
   [junit4]   2> 172645 T2609 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {a_t=PostingsFormat(name=LuceneVarGapFixedInterval), a_i=PostingsFormat(name=LuceneVarGapDocFreqInterval), id=PostingsFormat(name=LuceneVarGapDocFreqInterval), rnd_b=PostingsFormat(name=LuceneVarGapDocFreqInterval), _version_=PostingsFormat(name=LuceneVarGapFixedInterval)}, docValues:{}, sim=DefaultSimilarity, locale=ja, timezone=Canada/Saskatchewan
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=75957512,total=310378496
   [junit4]   2> NOTE: All tests run in this JVM: [DeleteInactiveReplicaTest, TestFuzzyAnalyzedSuggestions, DeleteShardTest, TestRebalanceLeaders, SchemaVersionSpecificBehaviorTest, SuggesterTest, ConnectionManagerTest, DistribJoinFromCollectionTest, TestCloudSchemaless, TestPivotHelperCode, SortByFunctionTest, UUIDFieldTest, TestReqParamsAPI, TestSimpleTrackingShardHandler, TestSearchPerf, TestJoin, TestUpdate, CurrencyFieldOpenExchangeTest, BasicZkTest, DebugComponentTest, TestHashPartitioner, TestWordDelimiterFilterFactory, OutOfBoxZkACLAndCredentialsProvidersTest, UUIDUpdateProcessorFallbackTest, DirectUpdateHandlerOptimizeTest, ZkNodePropsTest, TestJsonRequest, NumericFieldsTest, TestSolrConfigHandlerConcurrent, HdfsSyncSliceTest, ActionThrottleTest, CacheHeaderTest, TestFieldTypeCollectionResource, TestRequestStatusCollectionAPI, TestBinaryField, TestNonDefinedSimilarityFactory, TestCharFilters, PingRequestHandlerTest, InfoHandlerTest, DirectSolrConnectionTest, ScriptEngineTest, ClusterStateUpdateTest, TestMergePolicyConfig, TestQueryTypes, HighlighterTest, DateRangeFieldTest, TestDefaultSearchFieldResource, TestClassNameShortening, TestJettySolrRunner, RecoveryZkTest]
   [junit4] Completed [153/481] on J3 in 173.44s, 1 test, 1 failure <<< FAILURES!

[...truncated 1047 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:484: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:229: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:510: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1434: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:991: There were test failures: 481 suites, 1911 tests, 1 failure, 62 ignored (22 assumptions)

Total time: 57 minutes 57 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 230896502 bytes
Compression is 0.0%
Took 42 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



Re: [JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2917 - Still Failing

Posted by Steve Rowe <sa...@gmail.com>.
I logged into the lucene slave and ran ant clean-jars as jenkins on each of the non-4.x workspaces containing *-1.7.6.jar, so we shouldn’t see this particular problem again - here’s the list of affected workspaces:

Lucene-Solr-Tests-trunk-Java7  (obsolete)
Lucene-Solr-SmokeRelease-trunk (disabled)
Lucene-Solr-Tests-5.x-Java7
Lucene-Solr-Tests-trunk-Java8  (disabled)
Lucene-Solr-Clover-trunk       (disabled)
Lucene-Solr-NightlyTests-trunk (disabled)

Steve

> On Apr 9, 2015, at 9:53 AM, Apache Jenkins Server <je...@builds.apache.org> wrote:
> 
> Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2917/
> 
> All tests passed
> 
> Build Log:
> [...truncated 28458 lines...]
> check-licenses:
>     [echo] License check under: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr
> [licenses] MISSING sha1 checksum file for: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/server/lib/ext/jcl-over-slf4j-1.7.6.jar
> [licenses] EXPECTED sha1 checksum file : /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/licenses/jcl-over-slf4j-1.7.6.jar.sha1
> 
> [licenses] MISSING sha1 checksum file for: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/server/lib/ext/jul-to-slf4j-1.7.6.jar
> [licenses] EXPECTED sha1 checksum file : /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/licenses/jul-to-slf4j-1.7.6.jar.sha1
> [licenses] MISSING sha1 checksum file for: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/server/lib/ext/slf4j-api-1.7.6.jar
> [licenses] EXPECTED sha1 checksum file : /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/licenses/slf4j-api-1.7.6.jar.sha1
> 
> [...truncated 3 lines...]
> BUILD FAILED
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:536: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:97: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:294: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/tools/custom-tasks.xml:62: License check failed. Check the logs.
> If you recently modified ivy-versions.properties or any module's ivy.xml,
> make sure you run "ant clean-jars jar-checksums" before running precommit.
> 
> Total time: 90 minutes 29 seconds
> Build step 'Invoke Ant' marked build as failure
> Archiving artifacts
> Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
> Archived 1 artifacts
> Archive block size is 32768
> Received 0 blocks and 464 bytes
> Compression is 0.0%
> Took 0.18 sec
> Recording test results
> Email was triggered for: Failure
> Sending email for trigger: Failure
> 
> 
> 
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2917 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2917/

All tests passed

Build Log:
[...truncated 28458 lines...]
check-licenses:
     [echo] License check under: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr
 [licenses] MISSING sha1 checksum file for: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/server/lib/ext/jcl-over-slf4j-1.7.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/licenses/jcl-over-slf4j-1.7.6.jar.sha1

 [licenses] MISSING sha1 checksum file for: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/server/lib/ext/jul-to-slf4j-1.7.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/licenses/jul-to-slf4j-1.7.6.jar.sha1
 [licenses] MISSING sha1 checksum file for: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/server/lib/ext/slf4j-api-1.7.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/licenses/slf4j-api-1.7.6.jar.sha1

[...truncated 3 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:97: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:294: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/tools/custom-tasks.xml:62: License check failed. Check the logs.
If you recently modified ivy-versions.properties or any module's ivy.xml,
make sure you run "ant clean-jars jar-checksums" before running precommit.

Total time: 90 minutes 29 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 1 artifacts
Archive block size is 32768
Received 0 blocks and 464 bytes
Compression is 0.0%
Took 0.18 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2916 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2916/

1 tests failed.
REGRESSION:  org.apache.solr.schema.TestCloudSchemaless.test

Error Message:
QUERY FAILED: xpath=/response/arr[@name='fields']/lst/str[@name='name'][.='newTestFieldInt448']  request=/schema/fields?wt=xml  response=<?xml version="1.0" encoding="UTF-8"?> <response>  <lst name="responseHeader">   <int name="status">0</int>   <int name="QTime">1873</int> </lst> <arr name="fields">   <lst>     <str name="name">_version_</str>     <str name="type">long</str>     <bool name="stored">true</bool>     <bool name="indexed">true</bool>   </lst>   <lst>     <str name="name">constantField</str>     <str name="type">tdouble</str>   </lst>   <lst>     <str name="name">id</str>     <str name="type">string</str>     <bool name="stored">true</bool>     <bool name="indexed">true</bool>     <bool name="required">true</bool>     <bool name="multiValued">false</bool>     <bool name="uniqueKey">true</bool>   </lst>   <lst>     <str name="name">newTestFieldInt0</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt1</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt10</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt100</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt101</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt102</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt103</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt104</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt105</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt106</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt107</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt108</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt109</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt11</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt110</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt111</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt112</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt113</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt114</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt115</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt116</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt117</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt118</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt119</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt12</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt120</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt121</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt122</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt123</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt124</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt125</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt126</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt127</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt128</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt129</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt13</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt130</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt131</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt132</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt133</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt134</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt135</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt136</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt137</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt138</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt139</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt14</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt140</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt141</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt142</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt143</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt144</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt145</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt146</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt147</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt148</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt149</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt15</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt150</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt151</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt152</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt153</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt154</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt155</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt156</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt157</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt158</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt159</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt16</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt160</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt161</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt162</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt163</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt164</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt165</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt166</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt167</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt168</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt169</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt17</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt170</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt171</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt172</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt173</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt174</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt175</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt176</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt177</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt178</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt179</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt18</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt180</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt181</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt182</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt183</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt184</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt185</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt186</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt187</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt188</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt189</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt19</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt190</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt191</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt192</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt193</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt194</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt195</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt196</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt197</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt198</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt199</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt2</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt20</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt200</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt201</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt202</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt203</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt204</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt205</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt206</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt207</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt208</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt209</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt21</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt210</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt211</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt212</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt213</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt214</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt215</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt216</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt217</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt218</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt219</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt22</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt220</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt221</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt222</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt223</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt224</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt225</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt226</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt227</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt228</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt229</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt23</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt230</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt231</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt232</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt233</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt234</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt235</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt236</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt237</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt238</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt239</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt24</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt240</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt241</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt242</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt243</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt244</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt245</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt246</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt247</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt248</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt249</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt25</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt250</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt251</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt252</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt253</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt254</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt255</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt256</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt257</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt258</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt259</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt26</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt260</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt261</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt262</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt263</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt264</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt265</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt266</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt267</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt268</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt269</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt27</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt270</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt271</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt272</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt273</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt274</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt275</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt276</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt277</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt278</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt279</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt28</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt280</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt281</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt282</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt283</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt284</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt285</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt286</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt287</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt288</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt289</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt29</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt290</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt291</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt292</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt293</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt294</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt295</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt296</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt297</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt298</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt299</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt3</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt30</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt300</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt301</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt302</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt303</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt304</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt305</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt306</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt307</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt308</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt309</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt31</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt310</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt311</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt312</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt313</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt314</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt315</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt316</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt317</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt318</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt319</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt32</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt320</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt321</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt322</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt323</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt324</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt325</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt326</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt327</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt328</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt329</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt33</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt330</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt331</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt332</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt333</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt334</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt335</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt336</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt337</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt338</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt339</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt34</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt340</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt341</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt342</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt343</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt344</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt345</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt346</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt347</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt348</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt349</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt35</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt350</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt351</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt352</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt353</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt354</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt355</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt356</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt357</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt358</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt359</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt36</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt360</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt361</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt362</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt363</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt364</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt365</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt366</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt367</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt368</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt369</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt37</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt370</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt371</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt372</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt373</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt374</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt375</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt376</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt377</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt378</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt379</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt38</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt380</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt381</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt382</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt383</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt384</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt385</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt386</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt387</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt388</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt389</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt39</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt390</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt391</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt392</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt393</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt394</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt395</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt396</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt397</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt398</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt399</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt4</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt40</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt400</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt401</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt402</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt403</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt404</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt405</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt406</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt407</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt408</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt409</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt41</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt410</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt411</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt412</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt413</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt414</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt415</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt416</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt417</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt418</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt419</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt42</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt420</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt421</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt422</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt423</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt424</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt425</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt426</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt427</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt428</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt429</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt43</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt430</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt431</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt432</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt433</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt434</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt435</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt436</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt437</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt438</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt439</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt44</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt440</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt441</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt442</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt443</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt444</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt445</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt446</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt447</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt449</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt45</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt46</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt47</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt48</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt49</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt5</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt50</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt51</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt52</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt53</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt54</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt55</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt56</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt57</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt58</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt59</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt6</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt60</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt61</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt62</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt63</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt64</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt65</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt66</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt67</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt68</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt69</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt7</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt70</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt71</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt72</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt73</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt74</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt75</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt76</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt77</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt78</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt79</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt8</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt80</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt81</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt82</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt83</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt84</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt85</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt86</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt87</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt88</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt89</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt9</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt90</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt91</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt92</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt93</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt94</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt95</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt96</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt97</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt98</str>     <str name="type">tlong</str>   </lst>   <lst>     <str name="name">newTestFieldInt99</str>     <str name="type">tlong</str>   </lst> </arr> </response> 

Stack Trace:
java.lang.AssertionError: QUERY FAILED: xpath=/response/arr[@name='fields']/lst/str[@name='name'][.='newTestFieldInt448']  request=/schema/fields?wt=xml  response=<?xml version="1.0" encoding="UTF-8"?>
<response>

<lst name="responseHeader">
  <int name="status">0</int>
  <int name="QTime">1873</int>
</lst>
<arr name="fields">
  <lst>
    <str name="name">_version_</str>
    <str name="type">long</str>
    <bool name="stored">true</bool>
    <bool name="indexed">true</bool>
  </lst>
  <lst>
    <str name="name">constantField</str>
    <str name="type">tdouble</str>
  </lst>
  <lst>
    <str name="name">id</str>
    <str name="type">string</str>
    <bool name="stored">true</bool>
    <bool name="indexed">true</bool>
    <bool name="required">true</bool>
    <bool name="multiValued">false</bool>
    <bool name="uniqueKey">true</bool>
  </lst>
  <lst>
    <str name="name">newTestFieldInt0</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt1</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt10</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt100</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt101</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt102</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt103</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt104</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt105</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt106</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt107</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt108</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt109</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt11</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt110</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt111</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt112</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt113</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt114</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt115</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt116</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt117</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt118</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt119</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt12</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt120</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt121</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt122</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt123</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt124</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt125</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt126</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt127</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt128</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt129</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt13</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt130</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt131</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt132</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt133</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt134</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt135</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt136</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt137</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt138</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt139</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt14</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt140</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt141</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt142</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt143</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt144</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt145</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt146</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt147</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt148</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt149</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt15</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt150</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt151</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt152</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt153</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt154</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt155</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt156</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt157</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt158</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt159</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt16</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt160</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt161</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt162</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt163</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt164</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt165</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt166</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt167</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt168</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt169</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt17</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt170</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt171</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt172</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt173</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt174</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt175</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt176</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt177</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt178</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt179</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt18</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt180</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt181</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt182</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt183</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt184</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt185</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt186</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt187</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt188</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt189</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt19</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt190</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt191</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt192</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt193</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt194</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt195</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt196</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt197</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt198</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt199</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt2</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt20</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt200</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt201</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt202</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt203</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt204</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt205</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt206</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt207</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt208</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt209</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt21</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt210</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt211</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt212</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt213</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt214</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt215</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt216</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt217</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt218</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt219</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt22</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt220</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt221</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt222</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt223</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt224</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt225</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt226</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt227</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt228</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt229</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt23</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt230</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt231</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt232</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt233</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt234</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt235</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt236</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt237</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt238</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt239</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt24</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt240</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt241</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt242</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt243</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt244</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt245</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt246</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt247</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt248</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt249</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt25</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt250</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt251</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt252</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt253</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt254</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt255</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt256</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt257</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt258</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt259</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt26</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt260</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt261</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt262</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt263</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt264</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt265</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt266</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt267</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt268</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt269</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt27</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt270</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt271</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt272</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt273</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt274</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt275</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt276</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt277</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt278</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt279</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt28</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt280</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt281</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt282</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt283</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt284</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt285</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt286</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt287</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt288</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt289</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt29</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt290</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt291</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt292</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt293</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt294</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt295</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt296</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt297</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt298</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt299</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt3</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt30</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt300</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt301</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt302</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt303</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt304</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt305</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt306</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt307</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt308</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt309</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt31</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt310</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt311</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt312</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt313</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt314</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt315</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt316</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt317</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt318</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt319</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt32</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt320</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt321</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt322</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt323</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt324</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt325</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt326</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt327</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt328</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt329</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt33</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt330</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt331</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt332</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt333</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt334</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt335</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt336</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt337</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt338</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt339</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt34</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt340</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt341</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt342</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt343</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt344</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt345</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt346</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt347</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt348</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt349</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt35</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt350</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt351</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt352</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt353</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt354</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt355</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt356</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt357</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt358</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt359</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt36</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt360</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt361</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt362</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt363</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt364</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt365</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt366</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt367</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt368</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt369</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt37</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt370</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt371</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt372</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt373</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt374</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt375</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt376</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt377</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt378</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt379</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt38</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt380</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt381</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt382</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt383</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt384</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt385</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt386</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt387</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt388</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt389</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt39</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt390</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt391</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt392</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt393</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt394</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt395</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt396</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt397</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt398</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt399</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt4</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt40</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt400</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt401</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt402</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt403</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt404</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt405</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt406</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt407</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt408</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt409</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt41</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt410</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt411</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt412</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt413</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt414</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt415</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt416</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt417</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt418</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt419</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt42</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt420</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt421</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt422</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt423</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt424</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt425</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt426</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt427</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt428</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt429</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt43</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt430</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt431</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt432</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt433</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt434</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt435</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt436</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt437</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt438</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt439</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt44</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt440</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt441</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt442</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt443</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt444</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt445</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt446</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt447</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt449</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt45</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt46</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt47</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt48</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt49</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt5</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt50</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt51</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt52</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt53</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt54</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt55</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt56</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt57</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt58</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt59</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt6</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt60</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt61</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt62</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt63</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt64</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt65</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt66</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt67</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt68</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt69</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt7</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt70</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt71</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt72</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt73</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt74</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt75</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt76</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt77</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt78</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt79</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt8</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt80</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt81</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt82</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt83</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt84</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt85</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt86</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt87</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt88</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt89</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt9</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt90</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt91</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt92</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt93</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt94</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt95</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt96</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt97</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt98</str>
    <str name="type">tlong</str>
  </lst>
  <lst>
    <str name="name">newTestFieldInt99</str>
    <str name="type">tlong</str>
  </lst>
</arr>
</response>

	at __randomizedtesting.SeedInfo.seed([C8B1BCB8C8FBCD32:40E583626607A0CA]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.schema.TestCloudSchemaless.test(TestCloudSchemaless.java:159)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakCon

[...truncated too long message...]

4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt68</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt69</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt7</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt70</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt71</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt72</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt73</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt74</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt75</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt76</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt77</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt78</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt79</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt8</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt80</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt81</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt82</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt83</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt84</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt85</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt86</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt87</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt88</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt89</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt9</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt90</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt91</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt92</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt93</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt94</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt95</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt96</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt97</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt98</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst>
   [junit4]    >     <str name="name">newTestFieldInt99</str>
   [junit4]    >     <str name="type">tlong</str>
   [junit4]    >   </lst>
   [junit4]    > </arr>
   [junit4]    > </response>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([C8B1BCB8C8FBCD32:40E583626607A0CA]:0)
   [junit4]    > 	at org.apache.solr.schema.TestCloudSchemaless.test(TestCloudSchemaless.java:159)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1371084 T3547 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.schema.TestCloudSchemaless C8B1BCB8C8FBCD32-001
   [junit4]   2> 101046 T3546 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=true,coord=no): {}, locale=et, timezone=Pacific/Honolulu
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=142490696,total=348651520
   [junit4]   2> NOTE: All tests run in this JVM: [TestCoreDiscovery, TestBlobHandler, TestLuceneMatchVersion, TestDynamicLoading, TestDistributedSearch, ExternalCollectionsTest, TestDFRSimilarityFactory, DistributedQueryElevationComponentTest, TestSearcherReuse, ConnectionManagerTest, QueryResultKeyTest, DistribCursorPagingTest, TestBlendedInfixSuggestions, OpenCloseCoreStressTest, TestCloudPivotFacet, SolrCmdDistributorTest, DefaultValueUpdateProcessorTest, TestNonDefinedSimilarityFactory, PluginInfoTest, TestArbitraryIndexDir, TestPerFieldSimilarity, PrimitiveFieldTypeTest, TestRequestStatusCollectionAPI, TestTrie, LeaderInitiatedRecoveryOnCommitTest, HdfsChaosMonkeySafeLeaderTest, BasicFunctionalityTest, ResponseLogComponentTest, TestChildDocTransformer, TestStressUserVersions, TestReplicaProperties, TestImplicitCoreProperties, TestSurroundQueryParser, DistributedSuggestComponentTest, TestSolrQueryParser, SolrXmlInZkTest, BadIndexSchemaTest, TestBadConfig, IndexSchemaRuntimeFieldTest, TestJmxIntegration, TestReload, SpellCheckComponentTest, TestConfigSets, SyncSliceTest, TestRealTimeGet, ChangedSchemaMergeTest, CacheHeaderTest, TestCloudSchemaless]
   [junit4] Completed on J2 in 101.36s, 1 test, 1 failure <<< FAILURES!

[...truncated 763 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 481 suites, 1911 tests, 1 failure, 62 ignored (22 assumptions)

Total time: 57 minutes 51 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 206525730 bytes
Compression is 0.0%
Took 39 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2915 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2915/

1 tests failed.
REGRESSION:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
Didn't see all replicas for shard shard1 in collection1 come up within 30000 ms! ClusterState: {   "collection1":{     "shards":{"shard1":{         "range":"80000000-7fffffff",         "state":"active",         "replicas":{           "core_node1":{             "core":"collection1",             "state":"active",             "base_url":"http://127.0.0.1:30988/h/kg",             "node_name":"127.0.0.1:30988_h%2Fkg",             "leader":"true"},           "core_node2":{             "core":"collection1",             "state":"recovering",             "base_url":"http://127.0.0.1:45044/h/kg",             "node_name":"127.0.0.1:45044_h%2Fkg"}}}},     "maxShardsPerNode":"1",     "autoAddReplicas":"false",     "replicationFactor":"1",     "autoCreated":"true",     "router":{"name":"compositeId"}},   "control_collection":{     "shards":{"shard1":{         "range":"80000000-7fffffff",         "state":"active",         "replicas":{"core_node1":{             "core":"collection1",             "state":"active",             "base_url":"http://127.0.0.1:57657/h/kg",             "node_name":"127.0.0.1:57657_h%2Fkg",             "leader":"true"}}}},     "maxShardsPerNode":"1",     "autoAddReplicas":"false",     "replicationFactor":"1",     "autoCreated":"true",     "router":{"name":"compositeId"}}}

Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in collection1 come up within 30000 ms! ClusterState: {
  "collection1":{
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node1":{
            "core":"collection1",
            "state":"active",
            "base_url":"http://127.0.0.1:30988/h/kg",
            "node_name":"127.0.0.1:30988_h%2Fkg",
            "leader":"true"},
          "core_node2":{
            "core":"collection1",
            "state":"recovering",
            "base_url":"http://127.0.0.1:45044/h/kg",
            "node_name":"127.0.0.1:45044_h%2Fkg"}}}},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false",
    "replicationFactor":"1",
    "autoCreated":"true",
    "router":{"name":"compositeId"}},
  "control_collection":{
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node1":{
            "core":"collection1",
            "state":"active",
            "base_url":"http://127.0.0.1:57657/h/kg",
            "node_name":"127.0.0.1:57657_h%2Fkg",
            "leader":"true"}}}},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false",
    "replicationFactor":"1",
    "autoCreated":"true",
    "router":{"name":"compositeId"}}}
	at __randomizedtesting.SeedInfo.seed([6F561FCF85559A56:E70220152BA9F7AE]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:1920)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:102)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 9383 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/init-core-data-001
   [junit4]   2> 208180 T322 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /h/kg
   [junit4]   2> 208192 T322 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 208193 T323 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 208292 T322 oasc.ZkTestServer.run start zk server on port:48644
   [junit4]   2> 208293 T322 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 208294 T322 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 208299 T330 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5529d23e name:ZooKeeperConnection Watcher:127.0.0.1:48644 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 208300 T322 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 208301 T322 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 208302 T322 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 208306 T322 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 208308 T322 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 208310 T333 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@29f065a8 name:ZooKeeperConnection Watcher:127.0.0.1:48644/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 208310 T322 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 208311 T322 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 208312 T322 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 208316 T322 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 208318 T322 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 208320 T322 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 208325 T322 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 208326 T322 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 208330 T322 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 208331 T322 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 208334 T322 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 208336 T322 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 208339 T322 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 208340 T322 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 208343 T322 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 208344 T322 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 208346 T322 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 208347 T322 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 208350 T322 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 208351 T322 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 208353 T322 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 208354 T322 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 208356 T322 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 208357 T322 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 208360 T322 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 208361 T322 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 208364 T322 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 208364 T322 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 210925 T322 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores/collection1
   [junit4]   2> 210931 T322 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 210937 T322 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:58650
   [junit4]   2> 210937 T322 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/h/kg, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/tempDir-001/control/data, hostPort=57657}
   [junit4]   2> 210938 T322 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 210939 T322 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/'
   [junit4]   2> 211003 T322 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/solr.xml
   [junit4]   2> 211082 T322 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores
   [junit4]   2> 211082 T322 oasc.CoreContainer.<init> New CoreContainer 847984524
   [junit4]   2> 211083 T322 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/]
   [junit4]   2> 211083 T322 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/lib
   [junit4]   2> 211085 T322 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/lib).
   [junit4]   2> 211101 T322 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 211101 T322 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 211102 T322 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 211102 T322 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 211103 T322 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 211103 T322 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 211104 T322 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 211104 T322 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 211104 T322 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 211105 T322 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 211106 T322 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 211116 T322 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 211118 T322 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 211119 T322 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 211119 T322 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 211119 T322 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:48644/solr
   [junit4]   2> 211120 T322 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 211121 T322 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 211122 T322 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 211124 T345 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7b33f19f name:ZooKeeperConnection Watcher:127.0.0.1:48644 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 211125 T322 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 211126 T322 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 211129 T322 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 211131 T348 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@38433773 name:ZooKeeperConnection Watcher:127.0.0.1:48644/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 211132 T322 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 211134 T322 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 211136 T322 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 211139 T322 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 211141 T322 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 211143 T322 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 211145 T322 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 211147 T322 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 211149 T322 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 211150 T322 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57657_h%2Fkg
   [junit4]   2> 211151 T322 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:57657_h%2Fkg
   [junit4]   2> 211153 T322 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 211155 T322 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 211156 T322 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 211158 T322 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:57657_h%2Fkg
   [junit4]   2> 211158 T322 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 211160 T322 oasc.Overseer.start Overseer (id=93621668344889347-127.0.0.1:57657_h%2Fkg-n_0000000000) starting
   [junit4]   2> 211163 T322 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 211169 T322 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 211170 T350 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 211170 T349 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 211170 T322 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 211175 T322 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores
   [junit4]   2> 211183 T322 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores/collection1, shard=, coreNodeName=, name=collection1, dataDir=data/, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores/collection1/, config=solrconfig.xml, collection=control_collection, transient=false, loadOnStartup=true}
   [junit4]   2> 211183 T322 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores/collection1/
   [junit4]   2> 211184 T322 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 211185 T352 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 211185 T352 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 211187 T352 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 211187 T348 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 211188 T349 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57657/h/kg",
   [junit4]   2> 	  "node_name":"127.0.0.1:57657_h%2Fkg",
   [junit4]   2> 	  "operation":"state"} current state version: 0
   [junit4]   2> 211188 T349 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57657/h/kg",
   [junit4]   2> 	  "node_name":"127.0.0.1:57657_h%2Fkg",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 211189 T349 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 211189 T349 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 211190 T348 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 212187 T352 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 212188 T352 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 212188 T352 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 212189 T352 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 212189 T352 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 212190 T352 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores/collection1/'
   [junit4]   2> 212205 T352 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 212207 T352 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 212218 T352 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 212246 T352 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 212334 T352 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 212336 T352 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 212352 T352 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 212772 T352 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 212793 T352 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 212796 T352 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 212812 T352 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 212816 T352 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 212819 T352 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 212820 T352 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 212820 T352 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 212820 T352 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 212821 T352 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 212821 T352 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 212821 T352 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 212822 T352 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 212822 T352 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores/collection1/data/
   [junit4]   2> 212823 T352 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@62f98b99
   [junit4]   2> 212823 T352 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores/collection1/data
   [junit4]   2> 212824 T352 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores/collection1/data/index/
   [junit4]   2> 212824 T352 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 212824 T352 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/control-001/cores/collection1/data/index
   [junit4]   2> 212825 T352 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=21, maxMergeAtOnceExplicit=42, maxMergedSegmentMB=88.0810546875, floorSegmentMB=1.7578125, forceMergeDeletesPctAllowed=4.679093429411079, segmentsPerTier=15.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 212826 T352 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@348d830c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3696a3c8),segFN=segments_1,generation=1}
   [junit4]   2> 212826 T352 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 212840 T352 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 212840 T352 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 212841 T352 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 212841 T352 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 212841 T352 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 212842 T352 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 212842 T352 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 212842 T352 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 212843 T352 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 212855 T352 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 212857 T352 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 212858 T352 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 212860 T352 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 212868 T352 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/system,/admin/segments,/get,/update/json/docs,/update/csv,/admin/ping,/admin/file,/schema,/admin/properties,standard,/admin/luke,/config,/admin/mbeans,/update,/admin/logging,/admin/plugins,/update/json,/admin/threads,/replication
   [junit4]   2> 212874 T352 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 212876 T352 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 212876 T352 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 212877 T352 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 212877 T352 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 212878 T352 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=25, maxMergedSegmentMB=15.447265625, floorSegmentMB=1.6923828125, forceMergeDeletesPctAllowed=1.7328479877713976, segmentsPerTier=26.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 212879 T352 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@348d830c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3696a3c8),segFN=segments_1,generation=1}
   [junit4]   2> 212879 T352 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 212880 T352 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@41dbc868[collection1] main
   [junit4]   2> 212881 T352 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 212881 T352 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 212881 T352 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 212882 T352 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 212882 T352 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 212882 T352 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 212883 T352 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 212883 T352 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 212883 T352 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 212884 T352 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 212885 T353 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@41dbc868[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 212885 T352 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 212886 T356 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:57657/h/kg collection:control_collection shard:shard1
   [junit4]   2> 212886 T322 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 212887 T322 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 212887 T356 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 212889 T322 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 212890 T322 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 212891 T360 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3b182ea8 name:ZooKeeperConnection Watcher:127.0.0.1:48644/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 212891 T356 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 212891 T322 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 212892 T322 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 212893 T322 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 212893 T348 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 212893 T356 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 212894 T356 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 212894 T349 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C27 name=collection1 org.apache.solr.core.SolrCore@289cee0a url=http://127.0.0.1:57657/h/kg/collection1 node=127.0.0.1:57657_h%2Fkg C27_STATE=coll:control_collection core:collection1 props:{core=collection1, state=down, base_url=http://127.0.0.1:57657/h/kg, node_name=127.0.0.1:57657_h%2Fkg}
   [junit4]   2> 212894 T356 C:control_collection S:shard1 C27 P57657 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:57657/h/kg/collection1/
   [junit4]   2> 212895 T356 C:control_collection S:shard1 C27 P57657 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 212895 T356 C:control_collection S:shard1 C27 P57657 oasc.SyncStrategy.syncToMe http://127.0.0.1:57657/h/kg/collection1/ has no replicas
   [junit4]   2> 212895 T356 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:57657/h/kg/collection1/ shard1
   [junit4]   2> 212895 T356 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 212895 T360 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 212895 T322 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 212895 T348 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 212899 T348 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 212900 T349 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57657/h/kg",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 213003 T360 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 213003 T348 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 213051 T356 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:57657/h/kg/collection1/ and leader is http://127.0.0.1:57657/h/kg/collection1/
   [junit4]   2> 213052 T356 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:57657/h/kg
   [junit4]   2> 213052 T356 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 213053 T356 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 213053 T356 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 213055 T348 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 213056 T349 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57657/h/kg",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:57657_h%2Fkg",
   [junit4]   2> 	  "operation":"state"} current state version: 3
   [junit4]   2> 213056 T349 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57657/h/kg",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:57657_h%2Fkg",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 213160 T348 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 213160 T360 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 214571 T322 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores/collection1
   [junit4]   2> 214574 T322 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001
   [junit4]   2> 214574 T322 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 214577 T322 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:33918
   [junit4]   2> 214577 T322 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solrconfig=solrconfig.xml, hostPort=30988, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/tempDir-001/jetty1, hostContext=/h/kg, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores}
   [junit4]   2> 214578 T322 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 214578 T322 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/'
   [junit4]   2> 214650 T322 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/solr.xml
   [junit4]   2> 214715 T322 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores
   [junit4]   2> 214715 T322 oasc.CoreContainer.<init> New CoreContainer 1852987774
   [junit4]   2> 214715 T322 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/]
   [junit4]   2> 214716 T322 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/lib
   [junit4]   2> 214719 T322 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/lib).
   [junit4]   2> 214733 T322 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 214733 T322 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 214734 T322 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 214734 T322 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 214734 T322 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 214735 T322 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 214735 T322 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 214735 T322 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 214736 T322 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 214736 T322 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 214736 T322 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 214741 T322 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 214748 T322 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 214748 T322 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 214749 T322 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 214749 T322 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:48644/solr
   [junit4]   2> 214749 T322 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 214750 T322 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 214750 T322 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 214752 T372 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@670e068d name:ZooKeeperConnection Watcher:127.0.0.1:48644 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 214753 T322 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 214755 T322 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 214758 T322 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 214760 T375 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@549692e9 name:ZooKeeperConnection Watcher:127.0.0.1:48644/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 214760 T322 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 214766 T322 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 215771 T322 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:30988_h%2Fkg
   [junit4]   2> 215772 T322 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:30988_h%2Fkg
   [junit4]   2> 215775 T322 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 215778 T322 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores
   [junit4]   2> 215781 T322 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, schema=schema.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores/collection1/, loadOnStartup=true, transient=false, name=collection1, shard=, collection=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores/collection1, coreNodeName=, dataDir=data/}
   [junit4]   2> 215781 T322 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores/collection1/
   [junit4]   2> 215782 T322 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 215783 T376 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 215784 T376 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 215785 T376 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 215785 T348 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 215787 T349 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:30988/h/kg",
   [junit4]   2> 	  "node_name":"127.0.0.1:30988_h%2Fkg",
   [junit4]   2> 	  "operation":"state"} current state version: 4
   [junit4]   2> 215787 T349 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:30988/h/kg",
   [junit4]   2> 	  "node_name":"127.0.0.1:30988_h%2Fkg",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 215788 T349 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 215788 T349 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 215790 T360 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 215790 T348 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 215790 T375 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 216786 T376 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 216787 T376 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 216788 T376 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 216788 T376 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 216789 T376 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 216789 T376 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores/collection1/'
   [junit4]   2> 216808 T376 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 216810 T376 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 216831 T376 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 216851 T376 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 216892 T376 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 216893 T376 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 216910 T376 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 217450 T376 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 217471 T376 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 217475 T376 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 217498 T376 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 217503 T376 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 217508 T376 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 217509 T376 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 217510 T376 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 217510 T376 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 217511 T376 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 217511 T376 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 217512 T376 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 217512 T376 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 217513 T376 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 217513 T376 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@62f98b99
   [junit4]   2> 217514 T376 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores/collection1/data
   [junit4]   2> 217515 T376 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 217515 T376 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 217516 T376 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 217516 T376 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=21, maxMergeAtOnceExplicit=42, maxMergedSegmentMB=88.0810546875, floorSegmentMB=1.7578125, forceMergeDeletesPctAllowed=4.679093429411079, segmentsPerTier=15.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 217517 T376 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@749ce18a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@679b66d5),segFN=segments_1,generation=1}
   [junit4]   2> 217518 T376 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 217529 T376 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 217529 T376 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 217530 T376 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 217530 T376 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 217531 T376 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 217531 T376 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 217532 T376 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 217532 T376 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 217533 T376 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 217575 T376 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 217579 T376 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 217582 T376 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 217584 T376 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 217595 T376 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/system,/admin/segments,/get,/update/json/docs,/update/csv,/admin/ping,/admin/file,/schema,/admin/properties,standard,/admin/luke,/config,/admin/mbeans,/update,/admin/logging,/admin/plugins,/update/json,/admin/threads,/replication
   [junit4]   2> 217598 T376 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 217600 T376 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 217600 T376 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 217602 T376 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 217602 T376 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 217603 T376 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=25, maxMergedSegmentMB=15.447265625, floorSegmentMB=1.6923828125, forceMergeDeletesPctAllowed=1.7328479877713976, segmentsPerTier=26.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 217604 T376 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@749ce18a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@679b66d5),segFN=segments_1,generation=1}
   [junit4]   2> 217605 T376 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 217605 T376 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@56c72790[collection1] main
   [junit4]   2> 217606 T376 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 217607 T376 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 217607 T376 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 217608 T376 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 217608 T376 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 217609 T376 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 217610 T376 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 217610 T376 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 217610 T376 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 217611 T376 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 217612 T377 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@56c72790[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 217613 T376 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 217614 T380 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:30988/h/kg collection:collection1 shard:shard1
   [junit4]   2> 217614 T322 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 217615 T322 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 217615 T380 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 217620 T380 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 217622 T348 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 217622 T380 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 217623 T380 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 217623 T349 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> ASYNC  NEW_CORE C28 name=collection1 org.apache.solr.core.SolrCore@6f2ccccc url=http://127.0.0.1:30988/h/kg/collection1 node=127.0.0.1:30988_h%2Fkg C28_STATE=coll:collection1 core:collection1 props:{core=collection1, state=down, base_url=http://127.0.0.1:30988/h/kg, node_name=127.0.0.1:30988_h%2Fkg}
   [junit4]   2> 217623 T380 C:collection1 S:shard1 C28 P30988 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:30988/h/kg/collection1/
   [junit4]   2> 217624 T380 C:collection1 S:shard1 C28 P30988 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 217624 T380 C:collection1 S:shard1 C28 P30988 oasc.SyncStrategy.syncToMe http://127.0.0.1:30988/h/kg/collection1/ has no replicas
   [junit4]   2> 217624 T380 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:30988/h/kg/collection1/ shard1
   [junit4]   2> 217624 T380 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 217625 T360 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 217625 T375 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 217625 T348 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 217628 T348 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 217629 T349 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:30988/h/kg",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 217733 T360 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 217733 T375 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 217733 T348 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 217781 T380 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:30988/h/kg/collection1/ and leader is http://127.0.0.1:30988/h/kg/collection1/
   [junit4]   2> 217782 T380 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:30988/h/kg
   [junit4]   2> 217782 T380 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 217783 T380 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 217783 T380 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 217785 T348 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 217786 T349 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:30988/h/kg",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:30988_h%2Fkg",
   [junit4]   2> 	  "operation":"state"} current state version: 7
   [junit4]   2> 217787 T349 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:30988/h/kg",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:30988_h%2Fkg",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 217891 T360 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 217892 T348 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 217891 T375 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 219542 T322 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores/collection1
   [junit4]   2> 219544 T322 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001
   [junit4]   2> 219545 T322 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 219548 T322 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:39770
   [junit4]   2> 219548 T322 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores, hostContext=/h/kg, hostPort=45044, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/tempDir-001/jetty2, solrconfig=solrconfig.xml}
   [junit4]   2> 219549 T322 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 219549 T322 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/'
   [junit4]   2> 219589 T322 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/solr.xml
   [junit4]   2> 219631 T322 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores
   [junit4]   2> 219632 T322 oasc.CoreContainer.<init> New CoreContainer 591304576
   [junit4]   2> 219632 T322 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/]
   [junit4]   2> 219632 T322 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/lib
   [junit4]   2> 219633 T322 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/lib).
   [junit4]   2> 219658 T322 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 219659 T322 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 219660 T322 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 219660 T322 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 219660 T322 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 219661 T322 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 219661 T322 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 219662 T322 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 219662 T322 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 219663 T322 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 219663 T322 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 219690 T322 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 219692 T322 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 219692 T322 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 219693 T322 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 219693 T322 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:48644/solr
   [junit4]   2> 219694 T322 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 219694 T322 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 219695 T322 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 219697 T393 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5fb8b872 name:ZooKeeperConnection Watcher:127.0.0.1:48644 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 219698 T322 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 219699 T322 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 219701 T322 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 219703 T396 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3155d7fc name:ZooKeeperConnection Watcher:127.0.0.1:48644/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 219703 T322 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 219709 T322 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 220714 T322 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45044_h%2Fkg
   [junit4]   2> 220715 T322 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:45044_h%2Fkg
   [junit4]   2> 220718 T322 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 220721 T322 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores
   [junit4]   2> 220724 T322 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {schema=schema.xml, loadOnStartup=true, transient=false, dataDir=data/, collection=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores/collection1, coreNodeName=, shard=, config=solrconfig.xml, name=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores/collection1/}
   [junit4]   2> 220725 T322 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores/collection1/
   [junit4]   2> 220726 T322 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 220727 T397 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 220728 T397 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 220729 T348 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 220729 T397 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 220730 T349 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:45044/h/kg",
   [junit4]   2> 	  "node_name":"127.0.0.1:45044_h%2Fkg",
   [junit4]   2> 	  "operation":"state"} current state version: 8
   [junit4]   2> 220730 T349 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:45044/h/kg",
   [junit4]   2> 	  "node_name":"127.0.0.1:45044_h%2Fkg",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 220731 T349 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 220731 T349 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 220732 T396 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 220732 T348 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 220732 T375 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 220732 T360 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 221730 T397 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 221731 T397 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 221732 T397 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 221732 T397 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 221732 T397 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 221733 T397 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores/collection1/'
   [junit4]   2> 221747 T397 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 221749 T397 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 221760 T397 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 221780 T397 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 221848 T397 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 221849 T397 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 221869 T397 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 222316 T397 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 222332 T397 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 222334 T397 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 222351 T397 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 222355 T397 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 222360 T397 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 222361 T397 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 222361 T397 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 222362 T397 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 222362 T397 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 222363 T397 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 222363 T397 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 222363 T397 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 222364 T397 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/sh

[...truncated too long message...]

tate.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 348207 T441 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 348211 T441 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 348240 T441 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 348240 T441 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores/collection1/data/index.20150409122150355 [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores/collection1/data/index.20150409122150355;done=false>>]
   [junit4]   2> 348241 T441 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores/collection1/data/index.20150409122150355
   [junit4]   2> 348241 T441 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores/collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores/collection1/data;done=false>>]
   [junit4]   2> 348242 T441 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001/shard-2-001/cores/collection1/data
   [junit4]   2> 348905 T442 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 348908 T322 C:control_collection S:shard1 c:collection1 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/h/kg,null}
   [junit4]   2> 349022 T322 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:48644 48644
   [junit4]   2> 349180 T323 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:48644 48644
   [junit4]   2> 349181 T323 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 349182 T322 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 6 connections to: http://127.0.0.1:30988/h/kg, target: http://127.0.0.1:33918/h/kg
   [junit4]   2> 349182 T322 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:57657/h/kg, target: http://127.0.0.1:58650/h/kg
   [junit4]   2> 349182 T322 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 1 connections to: http://127.0.0.1:45044/h/kg, target: http://127.0.0.1:39770/h/kg
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=6F561FCF85559A56 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=pt_PT -Dtests.timezone=Asia/Ulaanbaatar -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE  141s J3 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: Didn't see all replicas for shard shard1 in collection1 come up within 30000 ms! ClusterState: {
   [junit4]    >   "collection1":{
   [junit4]    >     "shards":{"shard1":{
   [junit4]    >         "range":"80000000-7fffffff",
   [junit4]    >         "state":"active",
   [junit4]    >         "replicas":{
   [junit4]    >           "core_node1":{
   [junit4]    >             "core":"collection1",
   [junit4]    >             "state":"active",
   [junit4]    >             "base_url":"http://127.0.0.1:30988/h/kg",
   [junit4]    >             "node_name":"127.0.0.1:30988_h%2Fkg",
   [junit4]    >             "leader":"true"},
   [junit4]    >           "core_node2":{
   [junit4]    >             "core":"collection1",
   [junit4]    >             "state":"recovering",
   [junit4]    >             "base_url":"http://127.0.0.1:45044/h/kg",
   [junit4]    >             "node_name":"127.0.0.1:45044_h%2Fkg"}}}},
   [junit4]    >     "maxShardsPerNode":"1",
   [junit4]    >     "autoAddReplicas":"false",
   [junit4]    >     "replicationFactor":"1",
   [junit4]    >     "autoCreated":"true",
   [junit4]    >     "router":{"name":"compositeId"}},
   [junit4]    >   "control_collection":{
   [junit4]    >     "shards":{"shard1":{
   [junit4]    >         "range":"80000000-7fffffff",
   [junit4]    >         "state":"active",
   [junit4]    >         "replicas":{"core_node1":{
   [junit4]    >             "core":"collection1",
   [junit4]    >             "state":"active",
   [junit4]    >             "base_url":"http://127.0.0.1:57657/h/kg",
   [junit4]    >             "node_name":"127.0.0.1:57657_h%2Fkg",
   [junit4]    >             "leader":"true"}}}},
   [junit4]    >     "maxShardsPerNode":"1",
   [junit4]    >     "autoAddReplicas":"false",
   [junit4]    >     "replicationFactor":"1",
   [junit4]    >     "autoCreated":"true",
   [junit4]    >     "router":{"name":"compositeId"}}}
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([6F561FCF85559A56:E70220152BA9F7AE]:0)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:1920)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:102)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 349231 T322 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 6F561FCF85559A56-001
   [junit4]   2> 141056 T321 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {timestamp=PostingsFormat(name=Asserting), range_facet_l=PostingsFormat(name=Asserting), a_t=PostingsFormat(name=Asserting), text=Lucene50(blocksize=128), id=PostingsFormat(name=Direct), multiDefault=PostingsFormat(name=Asserting), intDefault=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), _version_=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128)))}, docValues:{timestamp=DocValuesFormat(name=Asserting)}, sim=RandomSimilarityProvider(queryNorm=false,coord=no): {}, locale=pt_PT, timezone=Asia/Ulaanbaatar
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=103305600,total=232783872
   [junit4]   2> NOTE: All tests run in this JVM: [TestAnalyzeInfixSuggestions, TestElisionMultitermQuery, TestCustomSort, LeaderInitiatedRecoveryOnCommitTest, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J3 in 141.37s, 1 test, 1 failure <<< FAILURES!

[...truncated 1372 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 480 suites, 1910 tests, 1 failure, 62 ignored (22 assumptions)

Total time: 58 minutes 50 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 211615867 bytes
Compression is 0.0%
Took 44 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2914 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2914/

1 tests failed.
REGRESSION:  org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds

Error Message:
soft529 wasn't fast enough

Stack Trace:
java.lang.AssertionError: soft529 wasn't fast enough
	at __randomizedtesting.SeedInfo.seed([BB89623F8C2FAD9B:EA5D9BBF3D5C9D3C]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNotNull(Assert.java:526)
	at org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds(SoftAutoCommitTest.java:111)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 9433 lines...]
   [junit4] Suite: org.apache.solr.update.SoftAutoCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001
   [junit4]   2> 30731 T27 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (true)
   [junit4]   2> 30733 T27 oas.SolrTestCaseJ4.initCore ####initCore
   [junit4]   2> 30734 T27 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 30735 T27 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 30736 T27 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 30737 T27 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 30909 T27 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 30986 T27 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 31119 T27 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 31121 T27 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 31215 T27 oass.IndexSchema.readSchema [null] Schema name=test
   [junit4]   2> 32831 T27 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 32884 T27 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 32891 T27 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 32943 T27 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 32949 T27 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 32954 T27 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 32960 T27 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 32960 T27 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 32961 T27 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 32962 T27 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 32962 T27 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 32991 T27 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 32991 T27 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr
   [junit4]   2> 32992 T27 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/'
   [junit4]   2> 33037 T27 oasc.CoreContainer.<init> New CoreContainer 1465666313
   [junit4]   2> 33037 T27 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/]
   [junit4]   2> 33038 T27 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/lib
   [junit4]   2> 33038 T27 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/lib).
   [junit4]   2> 33057 T27 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 600000
   [junit4]   2> 33057 T27 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 33058 T27 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 60000
   [junit4]   2> 33058 T27 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 33058 T27 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 33059 T27 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 33059 T27 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 33060 T27 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 33060 T27 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 33061 T27 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 33061 T27 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 33070 T27 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=30000&connTimeout=30000&retry=true
   [junit4]   2> 33072 T27 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 33072 T27 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 33073 T27 oasc.CoreContainer.load Node Name: testNode
   [junit4]   2> 33076 T27 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, schema=schema.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, name=collection1, collection=collection1, instanceDir=collection1, transient=false, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001, loadOnStartup=true, shard=shard1}
   [junit4]   2> 33077 T28 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 33078 T28 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 33079 T28 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 33080 T28 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 33203 T28 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 33241 T28 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 33322 T28 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 33323 T28 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 33387 T28 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 33971 T28 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 34010 T28 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 34015 T28 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 34056 T28 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 34065 T28 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 34073 T28 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 34075 T28 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 34075 T28 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 34076 T28 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 34077 T28 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 34078 T28 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 34078 T28 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 34079 T28 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 34091 T28 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001/
   [junit4]   2> 34101 T28 c:collection1 oasc.JmxMonitoredMap.<init> No JMX servers found, not exposing Solr information with JMX.
   [junit4]   2> 34187 T28 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001
   [junit4]   2> 34188 T28 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001/index/
   [junit4]   2> 34188 T28 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001/index' doesn't exist. Creating new index...
   [junit4]   2> 34189 T28 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001/index
   [junit4]   2> 34190 T28 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=43, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 34199 T28 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b3f77f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73dd8233),segFN=segments_1,generation=1}
   [junit4]   2> 34200 T28 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 34614 T28 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 34751 T28 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 34761 T28 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 34762 T28 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 34762 T28 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 34763 T28 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 34763 T28 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 34816 T28 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 34816 T28 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 35079 T28 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 35080 T28 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 35081 T28 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 35108 T28 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 35110 T28 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 35112 T28 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 35114 T28 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 35373 T28 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 35480 T28 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /get,/admin/file,/admin/threads,standard,/search-facet-def,spellCheckCompRH1,spellCheckCompRH,/update/json/docs,/search-facet-invariants,/update/json,/update,dismax,/terms,mltrh,/replication,/admin/system,/schema,spellCheckCompRH_Direct,spellCheckWithWordbreak_Direct,/update/csv,/admin/plugins,/admin/ping,/admin/properties,/admin/segments,/admin/luke,defaults,/admin/logging,tvrh,/admin/mbeans,/config,lazy,/debug/dump,/mlt,spellCheckWithWordbreak,mock
   [junit4]   2> 35489 T28 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 35491 T28 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 35584 T28 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 35585 T28 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 35634 T28 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 35635 T28 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 35635 T28 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=14, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6545133281336737]
   [junit4]   2> 35637 T28 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b3f77f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73dd8233),segFN=segments_1,generation=1}
   [junit4]   2> 35637 T28 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 35638 T28 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@41341e7[collection1] main
   [junit4]   2> 35668 T28 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 35669 T28 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 35669 T28 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 35670 T28 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 35913 T28 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 36315 T28 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 36316 T29 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 36319 T29 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 36319 T29 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 36319 T29 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 36321 T29 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 36323 T29 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 36324 T29 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 36324 T28 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 36325 T29 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 36326 T29 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 36326 T29 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@41341e7[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 36346 T27 oas.SolrTestCaseJ4.initCore ####initCore end
   [junit4]   2> 36355 T27 oas.SolrTestCaseJ4.setUp ###Starting testSoftAndHardCommitMaxTimeMixedAdds
   [junit4]   2> 36356 T27 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 36357 T27 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 36358 T27 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 36359 T27 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 36537 T27 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 36575 T27 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 36648 T27 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 36649 T27 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 36706 T27 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 37138 T27 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 37161 T27 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 37163 T27 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 37186 T27 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 37192 T27 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 37196 T27 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 37197 T27 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 37198 T27 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 37198 T27 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 37199 T27 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 37200 T27 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 37200 T27 oasc.CoreContainer.reload Reloading SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 37201 T27 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001/
   [junit4]   2> 37201 T27 c:collection1 oasc.JmxMonitoredMap.<init> No JMX servers found, not exposing Solr information with JMX.
   [junit4]   2> 37202 T27 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001/index/
   [junit4]   2> 37206 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 37207 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 37208 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 37208 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 37208 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 37209 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 37209 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 37210 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 37210 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 37211 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 37212 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 37212 T27 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 37233 T27 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 37236 T27 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 37238 T27 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 37240 T27 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 37243 T27 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 37245 T27 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /get,/admin/file,/admin/threads,standard,/search-facet-def,spellCheckCompRH1,spellCheckCompRH,/update/json/docs,/search-facet-invariants,/update/json,/update,dismax,/terms,mltrh,/replication,/admin/system,/schema,spellCheckCompRH_Direct,spellCheckWithWordbreak_Direct,/update/csv,/admin/plugins,/admin/ping,/admin/properties,/admin/segments,/admin/luke,defaults,/admin/logging,tvrh,/admin/mbeans,/config,lazy,/debug/dump,/mlt,spellCheckWithWordbreak,mock
   [junit4]   2> 37247 T27 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 37250 T27 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 37256 T27 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 37257 T27 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 37258 T27 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@3dd2c849[collection1] main
   [junit4]   2> 37259 T27 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 37259 T27 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 37259 T27 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 37260 T27 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 37262 T27 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 37274 T27 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 37276 T31 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 37276 T27 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
   [junit4]   2> 37277 T27 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
   [junit4]   2> 37277 T27 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Closing old IndexWriter... core=collection1
   [junit4]   2> 37277 T31 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 37278 T31 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 37278 T27 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=20, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.858064534025872]
   [junit4]   2> 37278 T31 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 37280 T31 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 37281 T31 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 37282 T31 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 37283 T31 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 37284 T31 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 37285 T31 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3dd2c849[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 37287 T27 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b3f77f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73dd8233),segFN=segments_1,generation=1}
   [junit4]   2> 37287 T27 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 37288 T27 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
   [junit4]   2> 37288 T27 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@442cca32[collection1] main
   [junit4]   2> 37300 T27 c:collection1 oasc.CoreContainer.registerCore replacing core: collection1
   [junit4]   2> 37300 T27 c:collection1 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@367c10a8
   [junit4]   2> 37301 T31 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@442cca32[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 37301 T27 c:collection1 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=0,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=0,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=0,transaction_logs_total_number=0}
   [junit4]   2> 37302 T27 c:collection1 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> ASYNC  NEW_CORE C3 name=collection1 org.apache.solr.core.SolrCore@469681e0
   [junit4]   2> 38767 T27 c:collection1 C3 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[529 (1497931605250932736)]} 0 1458
   [junit4]   2> 39257 T33 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 39796 T27 c:collection1 oas.SolrTestCaseJ4.tearDown ###Ending testSoftAndHardCommitMaxTimeMixedAdds
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=SoftAutoCommitTest -Dtests.method=testSoftAndHardCommitMaxTimeMixedAdds -Dtests.seed=BB89623F8C2FAD9B -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=el_CY -Dtests.timezone=Europe/Luxembourg -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE 3.54s J2 | SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: soft529 wasn't fast enough
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([BB89623F8C2FAD9B:EA5D9BBF3D5C9D3C]:0)
   [junit4]    > 	at org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds(SoftAutoCommitTest.java:111)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 39893 T27 c:collection1 oas.SolrTestCaseJ4.setUp ###Starting testSoftAndHardCommitMaxTimeRapidAdds
   [junit4]   2> 39893 T27 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 39895 T27 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 39896 T27 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 39897 T27 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 39957 T32 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 40080 T27 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 40134 T27 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 40228 T27 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 40229 T27 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 40284 T33 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@4e096ef3[collection1] main
   [junit4]   2> 40285 T33 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 40286 T31 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4e096ef3[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):c1)))}
   [junit4]   2> 40287 T32 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b3f77f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73dd8233),segFN=segments_1,generation=1}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b3f77f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73dd8233),segFN=segments_2,generation=2}
   [junit4]   2> 40287 T32 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
   [junit4]   2> 40288 T32 c:collection1 oasc.SolrCore.openNewSearcher SolrIndexSearcher has not changed - not re-opening: org.apache.solr.search.SolrIndexSearcher
   [junit4]   2> 40289 T32 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 40321 T27 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 40879 T27 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 40915 T27 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 40920 T27 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 40955 T27 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 40963 T27 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 40970 T27 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 40972 T27 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 40973 T27 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 40973 T27 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 40975 T27 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 40975 T27 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 40976 T27 c:collection1 oasc.CoreContainer.reload Reloading SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 40977 T27 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001/
   [junit4]   2> 40978 T27 c:collection1 oasc.JmxMonitoredMap.<init> No JMX servers found, not exposing Solr information with JMX.
   [junit4]   2> 40978 T27 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001/index/
   [junit4]   2> 40983 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 40984 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 40985 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 40986 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 40986 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 40987 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 40988 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 40990 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 40990 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 40992 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 40992 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 40993 T27 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 41030 T27 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 41033 T27 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 41038 T27 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 41041 T27 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 41047 T27 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 41049 T27 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /get,/admin/file,/admin/threads,standard,/search-facet-def,spellCheckCompRH1,spellCheckCompRH,/update/json/docs,/search-facet-invariants,/update/json,/update,dismax,/terms,mltrh,/replication,/admin/system,/schema,spellCheckCompRH_Direct,spellCheckWithWordbreak_Direct,/update/csv,/admin/plugins,/admin/ping,/admin/properties,/admin/segments,/admin/luke,defaults,/admin/logging,tvrh,/admin/mbeans,/config,lazy,/debug/dump,/mlt,spellCheckWithWordbreak,mock
   [junit4]   2> 41052 T27 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 41055 T27 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 41056 T27 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 41056 T27 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 41058 T27 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@e611d79[collection1] main
   [junit4]   2> 41059 T27 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 41060 T27 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 41060 T27 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 41061 T27 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 41065 T27 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 41083 T27 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 41085 T34 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 41085 T27 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
   [junit4]   2> 41086 T27 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
   [junit4]   2> 41086 T27 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Closing old IndexWriter... core=collection1
   [junit4]   2> 41087 T34 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 41088 T34 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 41088 T34 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 41090 T34 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 41090 T27 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1831758711, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 41091 T34 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 41093 T34 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 41095 T34 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 41095 T27 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b3f77f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73dd8233),segFN=segments_2,generation=2}
   [junit4]   2> 41096 T27 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
   [junit4]   2> 41096 T34 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 41097 T27 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
   [junit4]   2> 41097 T34 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@e611d79[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):c1)))}
   [junit4]   2> 41103 T27 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@6067a790[collection1] main
   [junit4]   2> 41105 T27 c:collection1 oasc.CoreContainer.registerCore replacing core: collection1
   [junit4]   2> 41105 T34 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6067a790[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):c1)))}
   [junit4]   2> 41105 T27 c:collection1 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@469681e0
   [junit4]   2> 41108 T27 c:collection1 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=2,autocommit maxTime=1200ms,autocommits=1,soft autocommit maxTime=500ms,soft autocommits=1,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=1,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=143,transaction_logs_total_number=1}
   [junit4]   2> 41109 T27 c:collection1 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> ASYNC  NEW_CORE C4 name=collection1 org.apache.solr.core.SolrCore@14d97f8d
   [junit4]   2> 41117 T27 c:collection1 C4 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[5000 (1497931608497324032)]} 0 6
   [junit4]   2> 41122 T27 c:collection1 C4 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[5001 (1497931608506761216)]} 0 2
   [junit4]   2> 41126 T27 c:collection1 C4 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[5002 (1497931608512004096)]} 0 2
   [junit4]   2> 41130 T27 c:collection1 C4 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[5003 (1497931608516198400)]} 0 1
   [junit4]   2> 41135 T27 c:collection1 C4 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[5004 (1497931608520392704)]} 0 1
   [junit4]   2> 41137 T27 c:collection1 oas.SolrTestCaseJ4.tearDown ###Ending testSoftAndHardCommitMaxTimeRapidAdds
   [junit4]   2> 41152 T27 c:collection1 oas.SolrTestCaseJ4.setUp ###Starting testSoftAndHardCommitMaxTimeDelete
   [junit4]   2> 41152 T27 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 41155 T27 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 41156 T27 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 41157 T27 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 41313 T27 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 41363 T27 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 41458 T27 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 41459 T27 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 41498 T27 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 41618 T36 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 41783 T36 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@67d122a8[collection1] main
   [junit4]   2> 41784 T36 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 41784 T34 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@67d122a8[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):c1) Uninverting(_1(5.2.0):c5)))}
   [junit4]   2> 42043 T27 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 42095 T27 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 42100 T27 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 42130 T27 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 42142 T27 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 42148 T27 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 42149 T27 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 42150 T27 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 42151 T27 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 42152 T27 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 42152 T27 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 42153 T27 c:collection1 oasc.CoreContainer.reload Reloading SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 42154 T27 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001/
   [junit4]   2> 42155 T27 c:collection1 oasc.JmxMonitoredMap.<init> No JMX servers found, not exposing Solr information with JMX.
   [junit4]   2> 42156 T27 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001/index/
   [junit4]   2> 42160 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 42161 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 42162 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 42163 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 42163 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 42164 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 42165 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 42165 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 42166 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 42167 T27 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 42168 T27 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 42169 T27 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 42196 T27 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 42199 T27 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 42202 T27 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 42205 T27 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 42210 T27 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 42213 T27 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /get,/admin/file,/admin/threads,standard,/search-facet-def,spellCheckCompRH1,spellCheckCompRH,/update/json/docs,/search-facet-invariants,/update/json,/update,dismax,/terms,mltrh,/replication,/admin/system,/schema,spellCheckCompRH_Direct,spellCheckWithWordbreak_Direct,/update/csv,/admin/plugins,/admin/ping,/admin/properties,/admin/segments,/admin/luke,defaults,/admin/logging,tvrh,/admin/mbeans,/config,lazy,/debug/dump,/mlt,spellCheckWithWordbreak,mock
   [junit4]   2> 42216 T27 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 42218 T27 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 42224 T27 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 42225 T27 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 42226 T27 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@22bf06d6[collection1] main
   [junit4]   2> 42227 T27 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 42228 T27 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 42228 T27 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 42228 T27 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 42231 T27 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 42242 T27 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 42243 T37 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 42243 T27 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
   [junit4]   2> 42244 T27 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
   [junit4]   2> 42245 T27 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Closing old IndexWriter... core=collection1
   [junit4]   2> 42245 T37 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 42246 T37 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 42246 T27 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b3f77f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73dd8233),segFN=segments_2,generation=2}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b3f77f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73dd8233),segFN=segments_3,generation=3}
   [junit4]   2> 42246 T37 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 42247 T27 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
   [junit4]   2> 42248 T27 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=16, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7145006735137542]
   [junit4]   2> 42248 T37 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 42250 T37 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 42252 T37 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 42252 T27 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b3f77f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73dd8233),segFN=segments_3,generation=3}
   [junit4]   2> 42253 T27 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
   [junit4]   2> 42253 T27 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
   [junit4]   2> 42253 T37 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 42254 T27 c:collection1 oasc.SolrCore.getSearcher WARN [collection1] PERFORMANCE WARNING: Overlapping onDeckSearchers=2
   [junit4]   2> 42255 T37 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 42256 T37 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@22bf06d6[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):c1) Uninverting(_1(5.2.0):c5)))}
   [junit4]   2> 42260 T27 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@710b47d2[collection1] main
   [junit4]   2> 42261 T27 c:collection1 oasc.CoreContainer.registerCore replacing core: collection1
   [junit4]   2> 42261 T37 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@710b47d2[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):c1) Uninverting(_1(5.2.0):c5)))}
   [junit4]   2> 42262 T27 c:collection1 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@14d97f8d
   [junit4]   2> 42263 T27 c:collection1 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=1,autocommit maxTime=1200ms,autocommits=0,soft autocommit maxTime=500ms,soft autocommits=1,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=5,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=5,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=143,transaction_logs_total_number=1}
   [junit4]   2> 42264 T27 c:collection1 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> ASYNC  NEW_CORE C5 name=collection1 org.apache.solr.core.SolrCore@16552add
   [junit4]   2> 42268 T27 c:collection1 C5 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[529 (1497931609708429312)]} 0 3
   [junit4]   2> 42271 T27 c:collection1 C5 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 42304 T27 c:collection1 C5 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b3f77f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73dd8233),segFN=segments_3,generation=3}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b3f77f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73dd8233),segFN=segments_4,generation=4}
   [junit4]   2> 42304 T27 c:collection1 C5 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 4
   [junit4]   2> 42306 T27 c:collection1 C5 oass.SolrIndexSearcher.<init> Opening Searcher@28fbf6a1[collection1] main
   [junit4]   2> 42306 T27 c:collection1 C5 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 42307 T37 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@28fbf6a1[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_1(5.2.0):c5) Uninverting(_2(5.2.0):c1)))}
   [junit4]   2> 42309 T27 c:collection1 C5 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 38
   [junit4]   2> 42315 T27 c:collection1 C5 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {delete=[529 (-1497931609756663808)]} 0 4
   [junit4]   2> 42816 T39 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 42826 T39 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@7a3335c5[collection1] main
   [junit4]   2> 42827 T39 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 42827 T37 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7a3335c5[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_1(5.2.0):c5)))}
   [junit4]   2> 42830 T27 c:collection1 C5 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[550 (1497931610297729024)]} 0 3
   [junit4]   2> 43331 T39 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 43349 T39 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@536fe809[collection1] main
   [junit4]   2> 43350 T39 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 43350 T37 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@536fe809[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_1(5.2.0):c5) Uninverting(_3(5.2.0):c1)))}
   [junit4]   2> 43516 T38 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 43518 T38 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b3f77f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73dd8233),segFN=segments_4,generation=4}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b3f77f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73dd8233),segFN=segments_5,generation=5}
   [junit4]   2> 43518 T38 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 5
   [junit4]   2> 43520 T38 c:collection1 oasc.SolrCore.openNewSearcher SolrIndexSearcher has not changed - not re-opening: org.apache.solr.search.SolrIndexSearcher
   [junit4]   2> 43520 T38 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 45521 T27 c:collection1 oas.SolrTestCaseJ4.tearDown ###Ending testSoftAndHardCommitMaxTimeDelete
   [junit4]   2> 45523 T27 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> 45524 T27 c:collection1 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=1465666313
   [junit4]   2> 45525 T27 c:collection1 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@16552add
   [junit4]   2> 45526 T27 c:collection1 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=4,autocommit maxTime=1200ms,autocommits=1,soft autocommit maxTime=500ms,soft autocommits=2,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=2,cumulative_deletesById=1,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=707,transaction_logs_total_number=3}
   [junit4]   2> 45527 T27 c:collection1 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 45528 T27 c:collection1 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 45529 T27 c:collection1 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 45530 T27 c:collection1 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 45533 T27 c:collection1 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 45533 T27 c:collection1 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001 [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001;done=false>>]
   [junit4]   2> 45534 T27 c:collection1 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001
   [junit4]   2> 45535 T27 c:collection1 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001/index [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001/index;done=false>>]
   [junit4]   2> 45535 T27 c:collection1 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001/init-core-data-001/index
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest BB89623F8C2FAD9B-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50), sim=RandomSimilarityProvider(queryNorm=false,coord=no): {}, locale=el_CY, timezone=Europe/Luxembourg
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=210408840,total=255852544
   [junit4]   2> NOTE: All tests run in this JVM: [TestManagedSchemaFieldTypeResource, SoftAutoCommitTest]
   [junit4] Completed on J2 in 15.17s, 3 tests, 1 failure <<< FAILURES!

[...truncated 1512 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 480 suites, 1910 tests, 1 failure, 62 ignored (22 assumptions)

Total time: 60 minutes 25 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 221032214 bytes
Compression is 0.0%
Took 39 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2913 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2913/

2 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest

Error Message:
ERROR: SolrIndexSearcher opens=8 closes=7

Stack Trace:
java.lang.AssertionError: ERROR: SolrIndexSearcher opens=8 closes=7
	at __randomizedtesting.SeedInfo.seed([4868EF3346E25AEB]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.SolrTestCaseJ4.endTrackingSearchers(SolrTestCaseJ4.java:496)
	at org.apache.solr.SolrTestCaseJ4.afterClass(SolrTestCaseJ4.java:232)
	at sun.reflect.GeneratedMethodAccessor35.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:799)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest

Error Message:
2 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest:     1) Thread[id=3276, name=searcherExecutor-2038-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=3262, name=qtp1606563175-3262, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:503)         at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient.blockUntilFinished(ConcurrentUpdateSolrClient.java:389)         at org.apache.solr.update.StreamingSolrClients.blockUntilFinished(StreamingSolrClients.java:103)         at org.apache.solr.update.SolrCmdDistributor.blockAndDoRetries(SolrCmdDistributor.java:229)         at org.apache.solr.update.SolrCmdDistributor.finish(SolrCmdDistributor.java:89)         at org.apache.solr.update.processor.DistributedUpdateProcessor.doFinish(DistributedUpdateProcessor.java:781)         at org.apache.solr.update.processor.DistributedUpdateProcessor.finish(DistributedUpdateProcessor.java:1643)         at org.apache.solr.update.processor.LogUpdateProcessor.finish(LogUpdateProcessorFactory.java:183)         at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:83)         at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:143)         at org.apache.solr.core.SolrCore.execute(SolrCore.java:1988)         at org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:828)         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:446)         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)         at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:102)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)         at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)         at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)         at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)         at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)         at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)         at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)         at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)         at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)         at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)         at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)         at org.eclipse.jetty.server.Server.handle(Server.java:368)         at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)         at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)         at org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:953)         at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1014)         at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:953)         at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240)         at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)         at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest: 
   1) Thread[id=3276, name=searcherExecutor-2038-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=3262, name=qtp1606563175-3262, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:503)
        at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient.blockUntilFinished(ConcurrentUpdateSolrClient.java:389)
        at org.apache.solr.update.StreamingSolrClients.blockUntilFinished(StreamingSolrClients.java:103)
        at org.apache.solr.update.SolrCmdDistributor.blockAndDoRetries(SolrCmdDistributor.java:229)
        at org.apache.solr.update.SolrCmdDistributor.finish(SolrCmdDistributor.java:89)
        at org.apache.solr.update.processor.DistributedUpdateProcessor.doFinish(DistributedUpdateProcessor.java:781)
        at org.apache.solr.update.processor.DistributedUpdateProcessor.finish(DistributedUpdateProcessor.java:1643)
        at org.apache.solr.update.processor.LogUpdateProcessor.finish(LogUpdateProcessorFactory.java:183)
        at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:83)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:143)
        at org.apache.solr.core.SolrCore.execute(SolrCore.java:1988)
        at org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:828)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:446)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:102)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
        at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
        at org.eclipse.jetty.server.Server.handle(Server.java:368)
        at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
        at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
        at org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:953)
        at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1014)
        at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:953)
        at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240)
        at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
        at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([4868EF3346E25AEB]:0)




Build Log:
[...truncated 10048 lines...]
   [junit4] Suite: org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/init-core-data-001
   [junit4]   2> 1172756 T3222 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 1172765 T3222 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1172765 T3223 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1172865 T3222 oasc.ZkTestServer.run start zk server on port:33268
   [junit4]   2> 1172866 T3222 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1172867 T3222 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1172873 T3230 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1a882b79 name:ZooKeeperConnection Watcher:127.0.0.1:33268 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1172873 T3222 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1172874 T3222 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1172874 T3222 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1172878 T3222 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1172879 T3222 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1172881 T3233 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5e5fc8b5 name:ZooKeeperConnection Watcher:127.0.0.1:33268/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1172881 T3222 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1172882 T3222 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1172882 T3222 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1172884 T3222 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1172886 T3222 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1172887 T3222 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1172888 T3222 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1172889 T3222 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1172891 T3222 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 1172892 T3222 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1172995 T3222 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1172996 T3222 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1172998 T3222 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1172998 T3222 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1173000 T3222 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1173000 T3222 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1173002 T3222 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1173002 T3222 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1173004 T3222 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1173005 T3222 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1173006 T3222 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1173007 T3222 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1173008 T3222 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1173009 T3222 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1173010 T3222 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1173011 T3222 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1173012 T3222 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1173013 T3222 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1174614 T3222 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores/collection1
   [junit4]   2> 1174621 T3222 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1174628 T3222 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:33280
   [junit4]   2> 1174628 T3222 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=33280, hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/tempDir-001/control/data}
   [junit4]   2> 1174629 T3222 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@749cd006
   [junit4]   2> 1174630 T3222 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/'
   [junit4]   2> 1174700 T3222 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/solr.xml
   [junit4]   2> 1174835 T3222 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores
   [junit4]   2> 1174836 T3222 oasc.CoreContainer.<init> New CoreContainer 930892105
   [junit4]   2> 1174837 T3222 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/]
   [junit4]   2> 1174837 T3222 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/lib
   [junit4]   2> 1174839 T3222 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/lib).
   [junit4]   2> 1174856 T3222 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1174857 T3222 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1174857 T3222 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1174858 T3222 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1174858 T3222 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1174859 T3222 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1174859 T3222 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1174860 T3222 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1174860 T3222 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1174861 T3222 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1174861 T3222 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1174879 T3222 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1174881 T3222 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1174882 T3222 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1174882 T3222 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1174883 T3222 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:33268/solr
   [junit4]   2> 1174883 T3222 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1174884 T3222 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1174885 T3222 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1174888 T3245 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1fd45422 name:ZooKeeperConnection Watcher:127.0.0.1:33268 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1174888 T3222 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1174889 T3222 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1174894 T3222 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1174897 T3248 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@65435e59 name:ZooKeeperConnection Watcher:127.0.0.1:33268/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1174897 T3222 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1174901 T3222 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1174904 T3222 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1174907 T3222 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1174910 T3222 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1174915 T3222 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1174917 T3222 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1174919 T3222 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1174921 T3222 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1174923 T3222 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33280_
   [junit4]   2> 1174925 T3222 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:33280_
   [junit4]   2> 1174927 T3222 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1174930 T3222 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1174932 T3222 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1174934 T3222 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:33280_
   [junit4]   2> 1174935 T3222 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1174937 T3222 oasc.Overseer.start Overseer (id=93619858301714435-127.0.0.1:33280_-n_0000000000) starting
   [junit4]   2> 1174939 T3222 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1174948 T3222 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 1174950 T3250 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1174950 T3222 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1174950 T3249 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1174957 T3222 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores
   [junit4]   2> 1174967 T3222 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {schema=schema.xml, config=solrconfig.xml, transient=false, coreNodeName=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores/collection1, name=collection1, shard=, loadOnStartup=true, collection=control_collection, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores/collection1/, dataDir=data/}
   [junit4]   2> 1174968 T3222 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores/collection1/
   [junit4]   2> 1174969 T3222 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1174970 T3252 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1174970 T3252 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1174972 T3248 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1174972 T3252 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1174973 T3249 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33280",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:33280_",
   [junit4]   2> 	  "roles":null} current state version: 0
   [junit4]   2> 1174974 T3249 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33280",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:33280_",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 1174974 T3249 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 1174974 T3249 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1174976 T3248 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1175972 T3252 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1175973 T3252 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1175974 T3252 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1175974 T3252 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1175976 T3252 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1175977 T3252 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores/collection1/'
   [junit4]   2> 1176004 T3252 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1176008 T3252 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1176031 T3252 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1176086 T3252 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1176176 T3252 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1176279 T3252 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1176291 T3252 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1176758 T3252 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1176761 T3252 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1176763 T3252 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1176768 T3252 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1176796 T3252 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1176797 T3252 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
   [junit4]   2> 1176801 T3252 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores/collection1/data/
   [junit4]   2> 1176802 T3252 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2a7655aa
   [junit4]   2> 1176806 T3252 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores/collection1/data
   [junit4]   2> 1176810 T3252 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores/collection1/data/index/
   [junit4]   2> 1176813 T3252 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1176815 T3252 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores/collection1/data/index
   [junit4]   2> 1176816 T3252 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=31, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=32.4326171875, floorSegmentMB=0.822265625, forceMergeDeletesPctAllowed=24.40693562206468, segmentsPerTier=33.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6199579108735648
   [junit4]   2> 1176826 T3252 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores/collection1/data/index,segFN=segments_1,generation=1}
   [junit4]   2> 1176826 T3252 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1176836 T3252 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1176837 T3252 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1176837 T3252 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1176837 T3252 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1176838 T3252 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1176838 T3252 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1176839 T3252 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1176839 T3252 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1176840 T3252 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1176903 T3252 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1176908 T3252 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1176914 T3252 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1176917 T3252 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1176924 T3252 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /config,/admin/threads,standard,/admin/logging,/admin/luke,/update/json,/update/json/docs,/admin/ping,/admin/file,/admin/mbeans,/admin/system,/update/csv,/admin/plugins,/replication,/get,/admin/properties,/update,/admin/segments,/schema
   [junit4]   2> 1176929 T3252 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1176930 T3252 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1176930 T3252 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1176934 T3252 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1176935 T3252 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1176937 T3252 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=36, maxMergeAtOnceExplicit=10, maxMergedSegmentMB=72.5986328125, floorSegmentMB=1.576171875, forceMergeDeletesPctAllowed=29.48913303940835, segmentsPerTier=31.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.892008954113685
   [junit4]   2> 1176939 T3252 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/control-001/cores/collection1/data/index,segFN=segments_1,generation=1}
   [junit4]   2> 1176940 T3252 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1176941 T3252 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@6cec9e83[collection1] main
   [junit4]   2> 1176941 T3252 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1176943 T3252 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1176943 T3252 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1176944 T3252 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1176944 T3252 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1176945 T3252 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1176945 T3252 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1176946 T3252 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1176946 T3252 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1176947 T3252 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1176948 T3253 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6cec9e83[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1176949 T3252 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1176952 T3256 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:33280 collection:control_collection shard:shard1
   [junit4]   2> 1176954 T3222 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 1176954 T3222 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1176955 T3256 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1176958 T3222 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1176959 T3222 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1176962 T3259 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@40d62081 name:ZooKeeperConnection Watcher:127.0.0.1:33268/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1176962 T3222 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1176963 T3222 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1176964 T3222 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1176970 T3222 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1176971 T3256 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1176974 T3248 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1176974 T3256 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1176974 T3256 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C532 name=collection1 org.apache.solr.core.SolrCore@66c2b8de url=http://127.0.0.1:33280/collection1 node=127.0.0.1:33280_ C532_STATE=coll:control_collection core:collection1 props:{core=collection1, state=down, base_url=http://127.0.0.1:33280, node_name=127.0.0.1:33280_}
   [junit4]   2> 1176975 T3256 C:control_collection S:shard1 C532 P33280 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:33280/collection1/
   [junit4]   2> 1176975 T3249 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 1176975 T3256 C:control_collection S:shard1 C532 P33280 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1176976 T3256 C:control_collection S:shard1 C532 P33280 oasc.SyncStrategy.syncToMe http://127.0.0.1:33280/collection1/ has no replicas
   [junit4]   2> 1176976 T3256 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:33280/collection1/ shard1
   [junit4]   2> 1176976 T3256 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1176978 T3259 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1176978 T3248 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1176984 T3248 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1176986 T3249 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33280",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 1177091 T3248 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1177092 T3259 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1177139 T3256 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:33280/collection1/ and leader is http://127.0.0.1:33280/collection1/
   [junit4]   2> 1177140 T3256 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:33280
   [junit4]   2> 1177140 T3256 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1177140 T3256 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1177140 T3256 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1177143 T3248 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1177145 T3249 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33280",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:33280_",
   [junit4]   2> 	  "roles":null} current state version: 3
   [junit4]   2> 1177146 T3249 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33280",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:33280_",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 1177251 T3248 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1177251 T3259 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1180447 T3222 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores/collection1
   [junit4]   2> 1180453 T3222 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001
   [junit4]   2> 1180454 T3222 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1180461 T3222 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:33284
   [junit4]   2> 1180461 T3222 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/tempDir-001/jetty1, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores, solrconfig=solrconfig.xml, hostContext=/, hostPort=33284}
   [junit4]   2> 1180462 T3222 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@749cd006
   [junit4]   2> 1180462 T3222 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/'
   [junit4]   2> 1180551 T3222 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/solr.xml
   [junit4]   2> 1180683 T3222 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores
   [junit4]   2> 1180683 T3222 oasc.CoreContainer.<init> New CoreContainer 863431378
   [junit4]   2> 1180684 T3222 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/]
   [junit4]   2> 1180684 T3222 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/lib
   [junit4]   2> 1180687 T3222 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/lib).
   [junit4]   2> 1180702 T3222 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1180702 T3222 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1180703 T3222 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1180703 T3222 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1180703 T3222 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1180704 T3222 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1180704 T3222 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1180705 T3222 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1180705 T3222 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1180705 T3222 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1180706 T3222 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1180725 T3222 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1180727 T3222 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1180727 T3222 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1180727 T3222 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1180728 T3222 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:33268/solr
   [junit4]   2> 1180728 T3222 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1180729 T3222 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1180730 T3222 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1180733 T3271 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@254dca1c name:ZooKeeperConnection Watcher:127.0.0.1:33268 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1180733 T3222 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1180734 T3222 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1180739 T3222 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1180741 T3274 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6a2ea17c name:ZooKeeperConnection Watcher:127.0.0.1:33268/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1180742 T3222 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1180751 T3222 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1181759 T3222 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33284_
   [junit4]   2> 1181760 T3222 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:33284_
   [junit4]   2> 1181764 T3222 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1181766 T3222 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores
   [junit4]   2> 1181768 T3222 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores/collection1/, schema=schema.xml, coreNodeName=, transient=false, shard=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores/collection1, config=solrconfig.xml, dataDir=data/, name=collection1, loadOnStartup=true}
   [junit4]   2> 1181769 T3222 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores/collection1/
   [junit4]   2> 1181769 T3222 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1181770 T3275 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1181770 T3275 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1181771 T3275 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1181771 T3248 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1181772 T3249 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33284",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:33284_",
   [junit4]   2> 	  "roles":null} current state version: 4
   [junit4]   2> 1181772 T3249 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33284",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:33284_",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 1181773 T3249 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 1181773 T3249 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1181774 T3274 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1181774 T3259 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1181774 T3248 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1182771 T3275 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1182772 T3275 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1182773 T3275 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1182773 T3275 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1182775 T3275 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1182776 T3275 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores/collection1/'
   [junit4]   2> 1182802 T3275 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1182807 T3275 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1182825 T3275 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1182868 T3275 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1182952 T3275 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1183054 T3275 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1183074 T3275 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1183668 T3275 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1183674 T3275 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1183677 T3275 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1183686 T3275 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1183718 T3275 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1183718 T3275 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
   [junit4]   2> 1183721 T3275 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 1183721 T3275 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2a7655aa
   [junit4]   2> 1183724 T3275 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores/collection1/data
   [junit4]   2> 1183725 T3275 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 1183726 T3275 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1183728 T3275 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 1183728 T3275 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=31, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=32.4326171875, floorSegmentMB=0.822265625, forceMergeDeletesPctAllowed=24.40693562206468, segmentsPerTier=33.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6199579108735648
   [junit4]   2> 1183735 T3275 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores/collection1/data/index,segFN=segments_1,generation=1}
   [junit4]   2> 1183736 T3275 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1183748 T3275 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1183748 T3275 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1183749 T3275 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1183749 T3275 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1183750 T3275 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1183750 T3275 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1183751 T3275 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1183751 T3275 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1183752 T3275 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1183811 T3275 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1183817 T3275 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1183822 T3275 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1183828 T3275 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1183844 T3275 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /config,/admin/threads,standard,/admin/logging,/admin/luke,/update/json,/update/json/docs,/admin/ping,/admin/file,/admin/mbeans,/admin/system,/update/csv,/admin/plugins,/replication,/get,/admin/properties,/update,/admin/segments,/schema
   [junit4]   2> 1183849 T3275 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1183850 T3275 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1183850 T3275 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1183854 T3275 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1183855 T3275 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1183858 T3275 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=36, maxMergeAtOnceExplicit=10, maxMergedSegmentMB=72.5986328125, floorSegmentMB=1.576171875, forceMergeDeletesPctAllowed=29.48913303940835, segmentsPerTier=31.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.892008954113685
   [junit4]   2> 1183860 T3275 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-1-001/cores/collection1/data/index,segFN=segments_1,generation=1}
   [junit4]   2> 1183861 T3275 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1183862 T3275 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@7b2b64a1[collection1] main
   [junit4]   2> 1183862 T3275 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1183863 T3275 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1183864 T3275 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1183864 T3275 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1183865 T3275 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1183865 T3275 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1183866 T3275 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1183866 T3275 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1183867 T3275 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1183867 T3275 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1183869 T3276 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7b2b64a1[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1183869 T3275 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1183871 T3279 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:33284 collection:collection1 shard:shard1
   [junit4]   2> 1183872 T3222 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 1183872 T3279 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1183872 T3222 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1183879 T3279 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1183883 T3248 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1183884 T3279 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1183884 T3279 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 1183884 T3249 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> ASYNC  NEW_CORE C533 name=collection1 org.apache.solr.core.SolrCore@6d5604b6 url=http://127.0.0.1:33284/collection1 node=127.0.0.1:33284_ C533_STATE=coll:collection1 core:collection1 props:{core=collection1, state=down, base_url=http://127.0.0.1:33284, node_name=127.0.0.1:33284_}
   [junit4]   2> 1183884 T3279 C:collection1 S:shard1 C533 P33284 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:33284/collection1/
   [junit4]   2> 1183885 T3279 C:collection1 S:shard1 C533 P33284 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1183885 T3279 C:collection1 S:shard1 C533 P33284 oasc.SyncStrategy.syncToMe http://127.0.0.1:33284/collection1/ has no replicas
   [junit4]   2> 1183885 T3279 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:33284/collection1/ shard1
   [junit4]   2> 1183886 T3279 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1183886 T3274 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1183886 T3259 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1183886 T3248 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1183891 T3248 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1183894 T3249 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33284",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 1183999 T3274 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1183999 T3259 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1183999 T3248 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1184044 T3279 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:33284/collection1/ and leader is http://127.0.0.1:33284/collection1/
   [junit4]   2> 1184045 T3279 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:33284
   [junit4]   2> 1184045 T3279 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1184045 T3279 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 1184046 T3279 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1184047 T3248 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1184048 T3249 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33284",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:33284_",
   [junit4]   2> 	  "roles":null} current state version: 7
   [junit4]   2> 1184049 T3249 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33284",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:33284_",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 1184154 T3248 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1184154 T3274 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1184154 T3259 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1187313 T3222 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-2-001/cores/collection1
   [junit4]   2> 1187318 T3222 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-2-001
   [junit4]   2> 1187319 T3222 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1187324 T3222 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:33289
   [junit4]   2> 1187325 T3222 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/tempDir-001/jetty2, hostPort=33289, solrconfig=solrconfig.xml, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-2-001/cores}
   [junit4]   2> 1187326 T3222 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@749cd006
   [junit4]   2> 1187327 T3222 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-2-001/'
   [junit4]   2> 1187444 T3222 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-2-001/solr.xml
   [junit4]   2> 1187581 T3222 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-2-001/cores
   [junit4]   2> 1187582 T3222 oasc.CoreContainer.<init> New CoreContainer 83748162
   [junit4]   2> 1187582 T3222 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-2-001/]
   [junit4]   2> 1187583 T3222 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-2-001/lib
   [junit4]   2> 1187586 T3222 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-2-001/lib).
   [junit4]   2> 1187605 T3222 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1187605 T3222 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1187606 T3222 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1187606 T3222 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1187606 T3222 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1187607 T3222 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1187607 T3222 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1187607 T3222 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1187608 T3222 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1187608 T3222 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1187608 T3222 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1187626 T3222 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1187629 T3222 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1187629 T3222 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1187630 T3222 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1187630 T3222 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:33268/solr
   [junit4]   2> 1187631 T3222 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1187631 T3222 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1187632 T3222 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1187636 T3291 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6a348bc2 name:ZooKeeperConnection Watcher:127.0.0.1:33268 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1187636 T3222 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1187637 T3222 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1187642 T3222 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1187644 T3294 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@16ee53d9 name:ZooKeeperConnection Watcher:127.0.0.1:33268/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1187644 T3222 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1187653 T3222 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1188661 T3222 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33289_
   [junit4]   2> 1188663 T3222 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:33289_
   [junit4]   2> 1188669 T3222 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1188674 T3222 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-2-001/cores
   [junit4]   2> 1188685 T3222 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-2-001/cores/collection1, collection=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 4868EF3346E25AEB-001/shard-2-001/cores/collection1/, schema=schema.xml, transient=false, config=solrconfig.xml, loadOnStartup=true, coreNodeName=, shard=, dataDir=data/}
   [junit4]   2> 1188686 T3222 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/sol

[...truncated too long message...]

rmat(compressionMode=DUMMY, chunkSize=8, maxDocsPerChunk=79, blockSize=578), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=DUMMY, chunkSize=8, blockSize=578)), sim=DefaultSimilarity, locale=sv, timezone=Mexico/BajaSur
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=123053320,total=283115520
   [junit4]   2> NOTE: All tests run in this JVM: [DistribDocExpirationUpdateProcessorTest, StatsComponentTest, TestReload, ExitableDirectoryReaderTest, TestSweetSpotSimilarityFactory, ParsingFieldUpdateProcessorsTest, TimeZoneUtilsTest, TestFieldCollectionResource, CollectionReloadTest, TestSerializedLuceneMatchVersion, TestStressUserVersions, SimpleFacetsTest, HighlighterMaxOffsetTest, TestLRUStatsCache, TestNRTOpen, TestJsonRequest, TestSolr4Spatial2, ZkNodePropsTest, SortByFunctionTest, TestJsonFacets, MoreLikeThisHandlerTest, HdfsUnloadDistributedZkTest, SuggesterFSTTest, TestJmxIntegration, DebugComponentTest, CSVRequestHandlerTest, TestStandardQParsers, TestJettySolrRunner, TestAddFieldRealTimeGet, HdfsChaosMonkeySafeLeaderTest, TestRandomMergePolicy, TermVectorComponentDistributedTest, DistributedQueryComponentCustomSortTest, TestManagedSynonymFilterFactory, TestZkChroot, SOLR749Test, BlockDirectoryTest, TestArbitraryIndexDir, SSLMigrationTest, CursorMarkTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, DeleteReplicaTest, TestSearchPerf, CollectionsAPIDistributedZkTest, TestSolrConfigHandlerConcurrent, DistribCursorPagingTest, RequiredFieldsTest, UUIDFieldTest, XmlUpdateRequestHandlerTest, RegexBoostProcessorTest, URLClassifyProcessorTest, AlternateDirectoryTest, TestCloudInspectUtil, TestSchemaManager, JSONWriterTest, TestManagedSchema, DistribJoinFromCollectionTest, BasicDistributedZk2Test, TestDistributedGrouping, ChaosMonkeyNothingIsSafeTest]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ChaosMonkeyNothingIsSafeTest -Dtests.seed=4868EF3346E25AEB -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=sv -Dtests.timezone=Mexico/BajaSur -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.00s J3 | ChaosMonkeyNothingIsSafeTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: ERROR: SolrIndexSearcher opens=8 closes=7
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([4868EF3346E25AEB]:0)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.endTrackingSearchers(SolrTestCaseJ4.java:496)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.afterClass(SolrTestCaseJ4.java:232)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest: 
   [junit4]    >    1) Thread[id=3276, name=searcherExecutor-2038-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    2) Thread[id=3262, name=qtp1606563175-3262, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at java.lang.Object.wait(Native Method)
   [junit4]    >         at java.lang.Object.wait(Object.java:503)
   [junit4]    >         at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient.blockUntilFinished(ConcurrentUpdateSolrClient.java:389)
   [junit4]    >         at org.apache.solr.update.StreamingSolrClients.blockUntilFinished(StreamingSolrClients.java:103)
   [junit4]    >         at org.apache.solr.update.SolrCmdDistributor.blockAndDoRetries(SolrCmdDistributor.java:229)
   [junit4]    >         at org.apache.solr.update.SolrCmdDistributor.finish(SolrCmdDistributor.java:89)
   [junit4]    >         at org.apache.solr.update.processor.DistributedUpdateProcessor.doFinish(DistributedUpdateProcessor.java:781)
   [junit4]    >         at org.apache.solr.update.processor.DistributedUpdateProcessor.finish(DistributedUpdateProcessor.java:1643)
   [junit4]    >         at org.apache.solr.update.processor.LogUpdateProcessor.finish(LogUpdateProcessorFactory.java:183)
   [junit4]    >         at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:83)
   [junit4]    >         at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:143)
   [junit4]    >         at org.apache.solr.core.SolrCore.execute(SolrCore.java:1988)
   [junit4]    >         at org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:828)
   [junit4]    >         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:446)
   [junit4]    >         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)
   [junit4]    >         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
   [junit4]    >         at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:102)
   [junit4]    >         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
   [junit4]    >         at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
   [junit4]    >         at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
   [junit4]    >         at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
   [junit4]    >         at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
   [junit4]    >         at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
   [junit4]    >         at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
   [junit4]    >         at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
   [junit4]    >         at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
   [junit4]    >         at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
   [junit4]    >         at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
   [junit4]    >         at org.eclipse.jetty.server.Server.handle(Server.java:368)
   [junit4]    >         at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
   [junit4]    >         at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
   [junit4]    >         at org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:953)
   [junit4]    >         at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1014)
   [junit4]    >         at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:953)
   [junit4]    >         at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240)
   [junit4]    >         at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
   [junit4]    >         at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([4868EF3346E25AEB]:0)
   [junit4] Completed on J3 in 487.36s, 1 test, 1 failure, 1 error <<< FAILURES!

[...truncated 701 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 480 suites, 1910 tests, 2 suite-level errors, 62 ignored (22 assumptions)

Total time: 59 minutes 45 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 210258778 bytes
Compression is 0.0%
Took 38 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2912 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2912/

All tests passed

Build Log:
[...truncated 28423 lines...]
check-licenses:
     [echo] License check under: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr
 [licenses] MISSING sha1 checksum file for: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/server/lib/ext/jcl-over-slf4j-1.7.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/licenses/jcl-over-slf4j-1.7.6.jar.sha1

 [licenses] MISSING sha1 checksum file for: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/server/lib/ext/jul-to-slf4j-1.7.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/licenses/jul-to-slf4j-1.7.6.jar.sha1
 [licenses] MISSING sha1 checksum file for: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/server/lib/ext/slf4j-api-1.7.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/licenses/slf4j-api-1.7.6.jar.sha1

[...truncated 3 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:90: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:256: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/tools/custom-tasks.xml:62: License check failed. Check the logs.
If you recently modified ivy-versions.properties or any module's ivy.xml,
make sure you run "ant clean-jars jar-checksums" before running precommit.

Total time: 87 minutes 37 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 1 artifacts
Archive block size is 32768
Received 0 blocks and 464 bytes
Compression is 0.0%
Took 12 ms
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2911 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2911/

1 tests failed.
FAILED:  org.apache.solr.cloud.TestRebalanceLeaders.test

Error Message:
No live SolrServers available to handle this request:[http://127.0.0.1:20802/zg, http://127.0.0.1:20781/zg, http://127.0.0.1:20767/zg, http://127.0.0.1:20792/zg, http://127.0.0.1:20743/zg]

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:20802/zg, http://127.0.0.1:20781/zg, http://127.0.0.1:20767/zg, http://127.0.0.1:20792/zg, http://127.0.0.1:20743/zg]
	at __randomizedtesting.SeedInfo.seed([ED4E1CF455B33B43:651A232EFB4F56BB]:0)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:349)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1067)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.TestRebalanceLeaders.issueCommands(TestRebalanceLeaders.java:281)
	at org.apache.solr.cloud.TestRebalanceLeaders.rebalanceLeaderTest(TestRebalanceLeaders.java:108)
	at org.apache.solr.cloud.TestRebalanceLeaders.test(TestRebalanceLeaders.java:74)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:20767/zg: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 {trace=java.lang.AssertionError
	at org.apache.solr.common.util.ByteUtils.UTF8toUTF16(ByteUtils.java:36)
	at org.apache.solr.common.util.ByteUtils.UTF8toUTF16(ByteUtils.java:64)
	at org.apache.solr.common.cloud.ZkStateReader.fromJSON(ZkStateReader.java:137)
	at org.apache.solr.common.cloud.ZkNodeProps.load(ZkNodeProps.java:92)
	at org.apache.solr.cloud.DistributedQueue.containsTaskWithRequestId(DistributedQueue.java:127)
	at org.apache.solr.handler.admin.CollectionsHandler.overseerCollectionQueueContains(CollectionsHandler.java:647)
	at org.apache.solr.handler.admin.CollectionsHandler.handleResponse(CollectionsHandler.java:669)
	at org.apache.solr.handler.admin.CollectionsHandler.handleResponse(CollectionsHandler.java:652)
	at org.apache.solr.handler.admin.CollectionsHandler.rejoinElection(CollectionsHandler.java:447)
	at org.apache.solr.handler.admin.CollectionsHandler.insurePreferredIsLeader(CollectionsHandler.java:362)
	at org.apache.solr.handler.admin.CollectionsHandler.handleBalanceLeaders(CollectionsHandler.java:269)
	at org.apache.solr.handler.admin.CollectionsHandler.handleRequestBody(CollectionsHandler.java:235)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:143)
	at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:782)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:282)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:102)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
	at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
	at org.eclipse.jetty.server.Server.handle(Server.java:368)
	at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
	at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
	at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)
	at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
	at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)
	at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
	at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
	at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
	at java.lang.Thread.run(Thread.java:745)
,code=500}</title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /zg/admin/collections. Reason:
<pre>    {trace=java.lang.AssertionError
	at org.apache.solr.common.util.ByteUtils.UTF8toUTF16(ByteUtils.java:36)
	at org.apache.solr.common.util.ByteUtils.UTF8toUTF16(ByteUtils.java:64)
	at org.apache.solr.common.cloud.ZkStateReader.fromJSON(ZkStateReader.java:137)
	at org.apache.solr.common.cloud.ZkNodeProps.load(ZkNodeProps.java:92)
	at org.apache.solr.cloud.DistributedQueue.containsTaskWithRequestId(DistributedQueue.java:127)
	at org.apache.solr.handler.admin.CollectionsHandler.overseerCollectionQueueContains(CollectionsHandler.java:647)
	at org.apache.solr.handler.admin.CollectionsHandler.handleResponse(CollectionsHandler.java:669)
	at org.apache.solr.handler.admin.CollectionsHandler.handleResponse(CollectionsHandler.java:652)
	at org.apache.solr.handler.admin.CollectionsHandler.rejoinElection(CollectionsHandler.java:447)
	at org.apache.solr.handler.admin.CollectionsHandler.insurePreferredIsLeader(CollectionsHandler.java:362)
	at org.apache.solr.handler.admin.CollectionsHandler.handleBalanceLeaders(CollectionsHandler.java:269)
	at org.apache.solr.handler.admin.CollectionsHandler.handleRequestBody(CollectionsHandler.java:235)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:143)
	at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:782)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:282)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:102)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
	at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
	at org.eclipse.jetty.server.Server.handle(Server.java:368)
	at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
	at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
	at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)
	at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
	at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)
	at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
	at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
	at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
	at java.lang.Thread.run(Thread.java:745)
,code=500}</pre></p>
<hr /><i><small>Powered by Jetty://</small></i>
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
</body>
</html>

	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:525)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:338)
	... 48 more




Build Log:
[...truncated 10082 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestRebalanceLeaders
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/init-core-data-001
   [junit4]   2> 1491198 T5962 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 1491198 T5962 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /zg/
   [junit4]   2> 1491203 T5962 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1491204 T5963 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1491304 T5962 oasc.ZkTestServer.run start zk server on port:20738
   [junit4]   2> 1491305 T5962 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1491306 T5962 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1491309 T5970 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@40885ca name:ZooKeeperConnection Watcher:127.0.0.1:20738 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1491309 T5962 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1491310 T5962 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1491310 T5962 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1491312 T5962 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1491313 T5962 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1491315 T5973 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7c823094 name:ZooKeeperConnection Watcher:127.0.0.1:20738/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1491315 T5962 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1491315 T5962 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1491316 T5962 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1491318 T5962 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1491319 T5962 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1491320 T5962 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1491322 T5962 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1491322 T5962 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1491325 T5962 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 1491325 T5962 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1491427 T5962 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1491427 T5962 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1491429 T5962 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1491430 T5962 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1491431 T5962 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1491432 T5962 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1491433 T5962 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1491434 T5962 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1491436 T5962 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1491436 T5962 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1491438 T5962 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1491438 T5962 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1491440 T5962 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1491441 T5962 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1491442 T5962 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1491443 T5962 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1491444 T5962 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1491445 T5962 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1493135 T5962 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores/collection1
   [junit4]   2> 1493140 T5962 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1493145 T5962 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:20743
   [junit4]   2> 1493145 T5962 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/zg, hostPort=20743, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores}
   [junit4]   2> 1493146 T5962 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 1493146 T5962 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/'
   [junit4]   2> 1493205 T5962 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/solr.xml
   [junit4]   2> 1493272 T5962 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores
   [junit4]   2> 1493273 T5962 oasc.CoreContainer.<init> New CoreContainer 671212554
   [junit4]   2> 1493273 T5962 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/]
   [junit4]   2> 1493273 T5962 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/lib
   [junit4]   2> 1493274 T5962 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/lib).
   [junit4]   2> 1493283 T5962 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1493284 T5962 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1493284 T5962 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1493284 T5962 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1493284 T5962 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1493285 T5962 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1493285 T5962 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1493285 T5962 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1493285 T5962 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1493285 T5962 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1493286 T5962 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1493293 T5962 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1493294 T5962 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1493294 T5962 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1493295 T5962 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1493295 T5962 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:20738/solr
   [junit4]   2> 1493295 T5962 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1493295 T5962 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1493296 T5962 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1493297 T5985 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3e33214f name:ZooKeeperConnection Watcher:127.0.0.1:20738 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1493298 T5962 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1493299 T5962 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1493301 T5962 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1493302 T5988 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6718d986 name:ZooKeeperConnection Watcher:127.0.0.1:20738/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1493302 T5962 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1493303 T5962 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1493304 T5962 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1493305 T5962 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1493307 T5962 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1493308 T5962 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1493309 T5962 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1493310 T5962 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1493311 T5962 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1493312 T5962 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:20743_zg
   [junit4]   2> 1493312 T5962 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:20743_zg
   [junit4]   2> 1493313 T5962 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1493314 T5962 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1493315 T5962 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1493316 T5962 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:20743_zg
   [junit4]   2> 1493316 T5962 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1493317 T5962 oasc.Overseer.start Overseer (id=93617987373367299-127.0.0.1:20743_zg-n_0000000000) starting
   [junit4]   2> 1493318 T5962 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1493322 T5962 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 1493323 T5990 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1493323 T5962 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1493323 T5989 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1493326 T5962 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores
   [junit4]   2> 1493328 T5962 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, dataDir=data/, shard=, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores/collection1, loadOnStartup=true, schema=schema.xml, transient=false, collection=control_collection, coreNodeName=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores/collection1/}
   [junit4]   2> 1493329 T5962 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores/collection1/
   [junit4]   2> 1493329 T5962 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1493330 T5992 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1493330 T5992 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1493330 T5988 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1493330 T5992 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1493331 T5989 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20743/zg",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:20743_zg",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state"} current state version: 0
   [junit4]   2> 1493331 T5989 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20743/zg",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:20743_zg",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1493332 T5989 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 1493332 T5989 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1493333 T5988 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1494331 T5992 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1494332 T5992 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1494332 T5992 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1494332 T5992 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1494333 T5992 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1494333 T5992 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores/collection1/'
   [junit4]   2> 1494346 T5992 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1494348 T5992 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1494355 T5992 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1494370 T5992 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1494439 T5992 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1494540 T5992 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1494549 T5992 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1494886 T5992 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1494888 T5992 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1494890 T5992 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1494893 T5992 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1494905 T5992 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1494906 T5992 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1494906 T5992 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores/collection1/data/
   [junit4]   2> 1494906 T5992 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4bc15184
   [junit4]   2> 1494907 T5992 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores/collection1/data
   [junit4]   2> 1494907 T5992 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores/collection1/data/index/
   [junit4]   2> 1494907 T5992 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1494907 T5992 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/control-001/cores/collection1/data/index
   [junit4]   2> 1494908 T5992 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=48, maxMergedSegmentMB=96.4521484375, floorSegmentMB=1.4296875, forceMergeDeletesPctAllowed=18.51724424491889, segmentsPerTier=41.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1494908 T5992 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4e06e62a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@26f0f76b),segFN=segments_1,generation=1}
   [junit4]   2> 1494909 T5992 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1494912 T5992 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1494913 T5992 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1494913 T5992 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1494913 T5992 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1494913 T5992 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1494913 T5992 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1494914 T5992 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1494914 T5992 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1494914 T5992 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1494926 T5992 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1494928 T5992 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1494929 T5992 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1494930 T5992 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1494934 T5992 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update/json,/admin/plugins,/update/json/docs,/update/csv,/admin/ping,/config,/admin/luke,/admin/system,/admin/threads,/schema,/admin/file,/update,/admin/mbeans,/admin/logging,standard,/admin/segments,/replication,/get,/admin/properties
   [junit4]   2> 1494935 T5992 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1494936 T5992 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1494936 T5992 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1494937 T5992 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1494937 T5992 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1494938 T5992 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=15, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 1494938 T5992 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4e06e62a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@26f0f76b),segFN=segments_1,generation=1}
   [junit4]   2> 1494938 T5992 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1494939 T5992 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@1268aa85[collection1] main
   [junit4]   2> 1494939 T5992 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1494940 T5992 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1494940 T5992 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1494940 T5992 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1494940 T5992 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1494940 T5992 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1494941 T5992 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1494941 T5992 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1494941 T5992 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1494941 T5992 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1494942 T5993 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1268aa85[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1494943 T5992 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1494943 T5996 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:20743/zg collection:control_collection shard:shard1
   [junit4]   2> 1494944 T5962 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 1494944 T5962 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1494944 T5996 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1494946 T5962 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1494947 T5962 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1494948 T5996 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1494948 T5999 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7a1acfd1 name:ZooKeeperConnection Watcher:127.0.0.1:20738/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1494949 T5962 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1494949 T5962 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1494950 T5962 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1494950 T5996 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1494950 T5996 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 1494950 T5988 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> ASYNC  NEW_CORE C2726 name=collection1 org.apache.solr.core.SolrCore@46237e29 url=http://127.0.0.1:20743/zg/collection1 node=127.0.0.1:20743_zg C2726_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:20743/zg, node_name=127.0.0.1:20743_zg, core=collection1, state=down}
   [junit4]   2> 1494950 T5996 C:control_collection S:shard1 C2726 P20743 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:20743/zg/collection1/
   [junit4]   2> 1494951 T5996 C:control_collection S:shard1 C2726 P20743 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1494951 T5996 C:control_collection S:shard1 C2726 P20743 oasc.SyncStrategy.syncToMe http://127.0.0.1:20743/zg/collection1/ has no replicas
   [junit4]   2> 1494951 T5989 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 1494951 T5996 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:20743/zg/collection1/ shard1
   [junit4]   2> 1494952 T5962 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1494952 T5996 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1494952 T5962 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 1494952 T5999 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1494952 T5962 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1494952 T5988 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1494954 T5962 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1494956 T6002 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4cf39acf name:ZooKeeperConnection Watcher:127.0.0.1:20738/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1494956 T5962 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1494956 T5962 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1494957 T5988 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1494958 T5989 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20743/zg",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 1494960 T5989 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"4",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 1494960 T5989 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 1494961 T5999 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1494961 T5988 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1494963 T5988 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1494963 T5989 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 1494964 T5999 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1494964 T5988 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1495008 T5996 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:20743/zg/collection1/ and leader is http://127.0.0.1:20743/zg/collection1/
   [junit4]   2> 1495009 T5996 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:20743/zg
   [junit4]   2> 1495009 T5996 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1495009 T5996 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1495009 T5996 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1495011 T5988 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1495011 T5989 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20743/zg",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:20743_zg",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"4",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state"} current state version: 4
   [junit4]   2> 1495012 T5989 oasco.ReplicaMutator.updateState Update state numShards=4 message={
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20743/zg",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:20743_zg",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"4",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1495115 T5988 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1495115 T5999 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1496198 T5962 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores/collection1
   [junit4]   2> 1496200 T5962 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001
   [junit4]   2> 1496201 T5962 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1496203 T5962 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:20767
   [junit4]   2> 1496204 T5962 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=20767, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/tempDir-001/jetty1, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores, hostContext=/zg, solrconfig=solrconfig.xml}
   [junit4]   2> 1496204 T5962 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 1496205 T5962 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/'
   [junit4]   2> 1496236 T5962 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/solr.xml
   [junit4]   2> 1496273 T5962 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores
   [junit4]   2> 1496274 T5962 oasc.CoreContainer.<init> New CoreContainer 211441626
   [junit4]   2> 1496274 T5962 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/]
   [junit4]   2> 1496274 T5962 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/lib
   [junit4]   2> 1496275 T5962 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/lib).
   [junit4]   2> 1496284 T5962 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1496285 T5962 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1496285 T5962 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1496285 T5962 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1496285 T5962 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1496286 T5962 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1496286 T5962 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1496286 T5962 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1496286 T5962 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1496286 T5962 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1496287 T5962 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1496291 T5962 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1496292 T5962 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1496293 T5962 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1496293 T5962 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1496293 T5962 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:20738/solr
   [junit4]   2> 1496293 T5962 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1496293 T5962 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1496294 T5962 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1496296 T6014 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@179bb877 name:ZooKeeperConnection Watcher:127.0.0.1:20738 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1496297 T5962 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1496297 T5962 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1496299 T5962 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1496300 T6017 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@9c04ef6 name:ZooKeeperConnection Watcher:127.0.0.1:20738/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1496300 T5962 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1496303 T5962 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1497307 T5962 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:20767_zg
   [junit4]   2> 1497308 T5962 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:20767_zg
   [junit4]   2> 1497312 T5962 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1497314 T5962 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores
   [junit4]   2> 1497319 T5962 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {schema=schema.xml, loadOnStartup=true, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores/collection1, dataDir=data/, shard=, coreNodeName=, collection=collection1, name=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores/collection1/, config=solrconfig.xml}
   [junit4]   2> 1497319 T5962 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores/collection1/
   [junit4]   2> 1497319 T5962 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1497321 T6018 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1497321 T6018 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1497323 T5988 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1497323 T6018 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 1497323 T6018 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 1497324 T5989 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20767/zg",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:20767_zg",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"4",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state"} current state version: 5
   [junit4]   2> 1497324 T6018 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 1497324 T5989 oasco.ReplicaMutator.updateState Update state numShards=4 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20767/zg",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:20767_zg",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"4",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1497324 T5989 oasco.ReplicaMutator.updateState Collection already exists with numShards=4
   [junit4]   2> 1497325 T5989 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 1497325 T6018 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 1497325 T6018 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1497326 T5989 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 1497326 T6017 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1497327 T6017 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 1498325 T6018 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1498326 T6018 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1498326 T6018 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1498326 T6018 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1498327 T6018 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1498327 T6018 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores/collection1/'
   [junit4]   2> 1498340 T6018 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1498342 T6018 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1498349 T6018 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1498364 T6018 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1498399 T6018 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1498500 T6018 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1498508 T6018 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1498770 T6018 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1498771 T6018 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1498773 T6018 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1498776 T6018 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1498788 T6018 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1498789 T6018 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1498789 T6018 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 1498789 T6018 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4bc15184
   [junit4]   2> 1498790 T6018 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores/collection1/data
   [junit4]   2> 1498791 T6018 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 1498791 T6018 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1498791 T6018 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 1498792 T6018 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=48, maxMergedSegmentMB=96.4521484375, floorSegmentMB=1.4296875, forceMergeDeletesPctAllowed=18.51724424491889, segmentsPerTier=41.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1498793 T6018 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@47337b54 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4e5ec51),segFN=segments_1,generation=1}
   [junit4]   2> 1498793 T6018 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1498800 T6018 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1498800 T6018 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1498801 T6018 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1498801 T6018 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1498801 T6018 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1498802 T6018 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1498802 T6018 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1498802 T6018 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1498803 T6018 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1498822 T6018 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1498825 T6018 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1498827 T6018 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1498830 T6018 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1498837 T6018 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update/json,/admin/plugins,/update/json/docs,/update/csv,/admin/ping,/config,/admin/luke,/admin/system,/admin/threads,/schema,/admin/file,/update,/admin/mbeans,/admin/logging,standard,/admin/segments,/replication,/get,/admin/properties
   [junit4]   2> 1498839 T6018 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1498841 T6018 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1498841 T6018 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1498843 T6018 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1498843 T6018 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1498844 T6018 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=15, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 1498845 T6018 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@47337b54 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4e5ec51),segFN=segments_1,generation=1}
   [junit4]   2> 1498845 T6018 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1498845 T6018 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@49167439[collection1] main
   [junit4]   2> 1498846 T6018 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1498846 T6018 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1498846 T6018 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1498847 T6018 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1498847 T6018 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1498847 T6018 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1498847 T6018 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1498848 T6018 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1498848 T6018 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1498848 T6018 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1498849 T6019 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@49167439[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1498849 T6018 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1498850 T6022 C:collection1 S:shard2 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:20767/zg collection:collection1 shard:shard2
   [junit4]   2> 1498850 T5962 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 1498851 T5962 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1498851 T6022 C:collection1 S:shard2 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 1498854 T6022 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 1498855 T5988 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1498855 T6022 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1498856 T6022 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C2727 name=collection1 org.apache.solr.core.SolrCore@2d6de342 url=http://127.0.0.1:20767/zg/collection1 node=127.0.0.1:20767_zg C2727_STATE=coll:collection1 core:collection1 props:{base_url=http://127.0.0.1:20767/zg, node_name=127.0.0.1:20767_zg, core=collection1, state=down}
   [junit4]   2> 1498856 T6022 C:collection1 S:shard2 C2727 P20767 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:20767/zg/collection1/
   [junit4]   2> 1498856 T5989 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 1498856 T6022 C:collection1 S:shard2 C2727 P20767 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1498857 T6022 C:collection1 S:shard2 C2727 P20767 oasc.SyncStrategy.syncToMe http://127.0.0.1:20767/zg/collection1/ has no replicas
   [junit4]   2> 1498857 T6022 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:20767/zg/collection1/ shard2
   [junit4]   2> 1498857 T6022 C:collection1 S:shard2 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 1498857 T5989 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 1498858 T6017 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1498858 T6017 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 1498860 T5989 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20767/zg",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 1498861 T5989 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 1498861 T6017 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1498862 T6017 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 1498863 T5988 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1498910 T6022 C:collection1 S:shard2 oasc.ZkController.register We are http://127.0.0.1:20767/zg/collection1/ and leader is http://127.0.0.1:20767/zg/collection1/
   [junit4]   2> 1498911 T6022 C:collection1 S:shard2 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:20767/zg
   [junit4]   2> 1498911 T6022 C:collection1 S:shard2 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1498911 T6022 C:collection1 S:shard2 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 1498911 T6022 C:collection1 S:shard2 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1498912 T5988 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1498913 T5989 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20767/zg",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:20767_zg",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"4",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "operation":"state"} current state version: 5
   [junit4]   2> 1498914 T5989 oasco.ReplicaMutator.updateState Update state numShards=4 message={
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20767/zg",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:20767_zg",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"4",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1498914 T6022 C:collection1 S:shard2 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 1498914 T5989 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 3
   [junit4]   2> 1498915 T6017 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1498915 T6017 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 4 
   [junit4]   2> 1499784 T5962 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-2-001/cores/collection1
   [junit4]   2> 1499786 T5962 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-2-001
   [junit4]   2> 1499787 T5962 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1499790 T5962 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:20781
   [junit4]   2> 1499790 T5962 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-2-001/cores, hostPort=20781, solrconfig=solrconfig.xml, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/tempDir-001/jetty2, hostContext=/zg}
   [junit4]   2> 1499790 T5962 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 1499791 T5962 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-2-001/'
   [junit4]   2> 1499822 T5962 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-2-001/solr.xml
   [junit4]   2> 1499865 T5962 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-2-001/cores
   [junit4]   2> 1499866 T5962 oasc.CoreContainer.<init> New CoreContainer 1551576367
   [junit4]   2> 1499866 T5962 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-2-001/]
   [junit4]   2> 1499866 T5962 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-2-001/lib
   [junit4]   2> 1499867 T5962 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001/shard-2-001/lib).
   [junit4]   2> 1499877 T5962 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1499877 T5962 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1499878 T5962 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1499878 T5962 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1499878 T5962 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1499878 T5962 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1499878 T5962 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1499879 T5962 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1499879 T5962 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1499879 T5962 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1499879 T5962 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1499885 T5962 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client w

[...truncated too long message...]

kStateReader.java:137)
   [junit4]    > 	at org.apache.solr.common.cloud.ZkNodeProps.load(ZkNodeProps.java:92)
   [junit4]    > 	at org.apache.solr.cloud.DistributedQueue.containsTaskWithRequestId(DistributedQueue.java:127)
   [junit4]    > 	at org.apache.solr.handler.admin.CollectionsHandler.overseerCollectionQueueContains(CollectionsHandler.java:647)
   [junit4]    > 	at org.apache.solr.handler.admin.CollectionsHandler.handleResponse(CollectionsHandler.java:669)
   [junit4]    > 	at org.apache.solr.handler.admin.CollectionsHandler.handleResponse(CollectionsHandler.java:652)
   [junit4]    > 	at org.apache.solr.handler.admin.CollectionsHandler.rejoinElection(CollectionsHandler.java:447)
   [junit4]    > 	at org.apache.solr.handler.admin.CollectionsHandler.insurePreferredIsLeader(CollectionsHandler.java:362)
   [junit4]    > 	at org.apache.solr.handler.admin.CollectionsHandler.handleBalanceLeaders(CollectionsHandler.java:269)
   [junit4]    > 	at org.apache.solr.handler.admin.CollectionsHandler.handleRequestBody(CollectionsHandler.java:235)
   [junit4]    > 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:143)
   [junit4]    > 	at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:782)
   [junit4]    > 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:282)
   [junit4]    > 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
   [junit4]    > 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:102)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
   [junit4]    > 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
   [junit4]    > 	at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
   [junit4]    > 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
   [junit4]    > 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
   [junit4]    > 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
   [junit4]    > 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
   [junit4]    > 	at org.eclipse.jetty.server.Server.handle(Server.java:368)
   [junit4]    > 	at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
   [junit4]    > 	at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
   [junit4]    > 	at org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:942)
   [junit4]    > 	at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1004)
   [junit4]    > 	at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)
   [junit4]    > 	at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
   [junit4]    > 	at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
   [junit4]    > 	at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
   [junit4]    > 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
   [junit4]    > 	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > ,code=500}</pre></p>
   [junit4]    > <hr /><i><small>Powered by Jetty://</small></i>
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    > </body>
   [junit4]    > </html>
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:525)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:338)
   [junit4]    > 	... 48 more
   [junit4]   2> 1592283 T5962 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.TestRebalanceLeaders ED4E1CF455B33B43-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=false,coord=no): {}, locale=it_CH, timezone=Asia/Calcutta
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=211655760,total=402653184
   [junit4]   2> NOTE: All tests run in this JVM: [TestSolrQueryParserDefaultOperatorResource, TestNRTOpen, WordBreakSolrSpellCheckerTest, TestSolrXml, TestUpdate, TestCloudInspectUtil, TestElisionMultitermQuery, SpatialHeatmapFacetsTest, TestFieldCollectionResource, TestDistributedSearch, AnalysisErrorHandlingTest, TestLazyCores, DistribDocExpirationUpdateProcessorTest, CloudMLTQParserTest, DistributedFacetPivotLongTailTest, TestObjectReleaseTracker, TestSolrQueryParserResource, TestDistributedGrouping, FieldAnalysisRequestHandlerTest, ShardSplitTest, TestPostingsSolrHighlighter, DocValuesTest, FieldMutatingUpdateProcessorTest, TestManagedSynonymFilterFactory, PreAnalyzedFieldTest, DistributedQueryElevationComponentTest, TestInfoStreamLogging, HdfsChaosMonkeySafeLeaderTest, DistributedExpandComponentTest, OpenCloseCoreStressTest, BadCopyFieldTest, ClusterStateTest, TestStressLucene, CoreAdminCreateDiscoverTest, DeleteShardTest, HdfsBasicDistributedZkTest, TestSolrDeletionPolicy1, TestReload, ZkSolrClientTest, SpellCheckCollatorTest, AutoCommitTest, SpatialRPTFieldTypeTest, TestSuggestSpellingConverter, TestReplicaProperties, SharedFSAutoReplicaFailoverUtilsTest, CoreMergeIndexesAdminHandlerTest, TestOmitPositions, DirectSolrSpellCheckerTest, TestStressUserVersions, SpatialFilterTest, ZkStateWriterTest, DeleteReplicaTest, HdfsDirectoryTest, BadComponentTest, SortByFunctionTest, TestCustomSort, SolrPluginUtilsTest, TestCloudManagedSchemaConcurrent, TestCloudSchemaless, LeaderInitiatedRecoveryOnCommitTest, ShardRoutingTest, TestRebalanceLeaders]
   [junit4] Completed on J2 in 101.12s, 1 test, 1 error <<< FAILURES!

[...truncated 673 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 480 suites, 1905 tests, 1 error, 62 ignored (22 assumptions)

Total time: 58 minutes 9 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 212847609 bytes
Compression is 0.0%
Took 41 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2910 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2910/

1 tests failed.
REGRESSION:  org.apache.lucene.search.join.TestJoinUtil.testRandomOrdinalsJoin

Error Message:
expected:<296> but was:<269>

Stack Trace:
java.lang.AssertionError: expected:<296> but was:<269>
	at __randomizedtesting.SeedInfo.seed([5214ECCDEBB46393:EDC9ED543B48769F]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.lucene.search.join.TestJoinUtil.assertTopDocs(TestJoinUtil.java:714)
	at org.apache.lucene.search.join.TestJoinUtil.testRandomOrdinalsJoin(TestJoinUtil.java:345)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 6851 lines...]
   [junit4] Suite: org.apache.lucene.search.join.TestJoinUtil
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestJoinUtil -Dtests.method=testRandomOrdinalsJoin -Dtests.seed=5214ECCDEBB46393 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=de_LU -Dtests.timezone=Europe/Jersey -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE 0.76s J1 | TestJoinUtil.testRandomOrdinalsJoin <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: expected:<296> but was:<269>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([5214ECCDEBB46393:EDC9ED543B48769F]:0)
   [junit4]    > 	at org.apache.lucene.search.join.TestJoinUtil.assertTopDocs(TestJoinUtil.java:714)
   [junit4]    > 	at org.apache.lucene.search.join.TestJoinUtil.testRandomOrdinalsJoin(TestJoinUtil.java:345)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {movieId=PostingsFormat(name=MockRandom), name=PostingsFormat(name=MockRandom), id=PostingsFormat(name=MockRandom), description=PostingsFormat(name=Memory doPackFST= true), type=PostingsFormat(name=LuceneVarGapFixedInterval), to=PostingsFormat(name=MockRandom), price=PostingsFormat(name=LuceneFixedGap), value=PostingsFormat(name=LuceneFixedGap), from=PostingsFormat(name=LuceneVarGapFixedInterval), subtitle=PostingsFormat(name=Memory doPackFST= true), productId=PostingsFormat(name=LuceneVarGapFixedInterval)}, docValues:{movieId=DocValuesFormat(name=Asserting), id=DocValuesFormat(name=Asserting), to=DocValuesFormat(name=Asserting), join_field=DocValuesFormat(name=Memory), idproductId=DocValuesFormat(name=Asserting), from=DocValuesFormat(name=Lucene50), productId=DocValuesFormat(name=Lucene50)}, sim=DefaultSimilarity, locale=de_LU, timezone=Europe/Jersey
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=151345944,total=272105472
   [junit4]   2> NOTE: All tests run in this JVM: [TestJoinUtil]
   [junit4] Completed on J1 in 15.11s, 9 tests, 1 failure <<< FAILURES!

[...truncated 12 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/build.xml:456: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:2166: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/module-build.xml:58: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 4 suites, 29 tests, 1 failure

Total time: 18 minutes 42 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 62472 bytes
Compression is 0.0%
Took 0.27 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2909 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2909/

All tests passed

Build Log:
[...truncated 28423 lines...]
check-licenses:
     [echo] License check under: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr
 [licenses] MISSING sha1 checksum file for: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/server/lib/ext/jcl-over-slf4j-1.7.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/licenses/jcl-over-slf4j-1.7.6.jar.sha1

 [licenses] MISSING sha1 checksum file for: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/server/lib/ext/jul-to-slf4j-1.7.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/licenses/jul-to-slf4j-1.7.6.jar.sha1
 [licenses] MISSING sha1 checksum file for: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/server/lib/ext/slf4j-api-1.7.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/licenses/slf4j-api-1.7.6.jar.sha1

[...truncated 3 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:90: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:256: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/tools/custom-tasks.xml:62: License check failed. Check the logs.
If you recently modified ivy-versions.properties or any module's ivy.xml,
make sure you run "ant clean-jars jar-checksums" before running precommit.

Total time: 93 minutes 35 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 1 artifacts
Archive block size is 32768
Received 0 blocks and 464 bytes
Compression is 0.0%
Took 0.2 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2908 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2908/

1 tests failed.
FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
java.net.SocketTimeoutException: Read timed out

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: java.net.SocketTimeoutException: Read timed out
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.sendDocsWithRetry(AbstractFullDistribZkTestBase.java:774)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:277)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 10147 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/init-core-data-001
   [junit4]   2> 1371559 T4624 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /nz_fss/la
   [junit4]   2> 1371564 T4624 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1371565 T4625 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1371666 T4624 oasc.ZkTestServer.run start zk server on port:25779
   [junit4]   2> 1371667 T4624 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1371668 T4624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1371673 T4632 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4e862073 name:ZooKeeperConnection Watcher:127.0.0.1:25779 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1371673 T4624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1371674 T4624 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1371674 T4624 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1371679 T4624 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1371680 T4624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1371681 T4635 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@268cfa3e name:ZooKeeperConnection Watcher:127.0.0.1:25779/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1371681 T4624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1371682 T4624 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1371682 T4624 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1371685 T4624 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1371686 T4624 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1371687 T4624 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1371689 T4624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1371689 T4624 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1371692 T4624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1371693 T4624 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1371796 T4624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1371797 T4624 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1371798 T4624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1371799 T4624 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1371801 T4624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1371801 T4624 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1371803 T4624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1371804 T4624 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1371805 T4624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1371806 T4624 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1371807 T4624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1371808 T4624 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1371810 T4624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1371810 T4624 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1371812 T4624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1371812 T4624 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1371814 T4624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1371815 T4624 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1373224 T4624 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores/collection1
   [junit4]   2> 1373227 T4624 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1373233 T4624 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:25783
   [junit4]   2> 1373233 T4624 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/nz_fss/la, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/tempDir-001/control/data, hostPort=25782, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores}
   [junit4]   2> 1373233 T4624 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 1373234 T4624 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/'
   [junit4]   2> 1373272 T4624 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/solr.xml
   [junit4]   2> 1373316 T4624 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores
   [junit4]   2> 1373316 T4624 oasc.CoreContainer.<init> New CoreContainer 1448812155
   [junit4]   2> 1373316 T4624 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/]
   [junit4]   2> 1373316 T4624 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/lib
   [junit4]   2> 1373317 T4624 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/lib).
   [junit4]   2> 1373326 T4624 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1373327 T4624 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1373327 T4624 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1373327 T4624 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1373327 T4624 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1373328 T4624 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1373328 T4624 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1373328 T4624 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1373328 T4624 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1373328 T4624 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1373329 T4624 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1373335 T4624 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1373335 T4624 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1373336 T4624 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1373336 T4624 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1373336 T4624 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:25779/solr
   [junit4]   2> 1373336 T4624 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1373337 T4624 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1373337 T4624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1373339 T4647 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@772c4729 name:ZooKeeperConnection Watcher:127.0.0.1:25779 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1373340 T4624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1373340 T4624 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1373342 T4624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1373343 T4650 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3548fe79 name:ZooKeeperConnection Watcher:127.0.0.1:25779/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1373343 T4624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1373345 T4624 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1373346 T4624 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1373348 T4624 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1373349 T4624 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1373350 T4624 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1373351 T4624 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1373352 T4624 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1373353 T4624 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1373354 T4624 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:25782_nz_fss%2Fla
   [junit4]   2> 1373354 T4624 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:25782_nz_fss%2Fla
   [junit4]   2> 1373355 T4624 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1373356 T4624 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1373357 T4624 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1373358 T4624 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:25782_nz_fss%2Fla
   [junit4]   2> 1373358 T4624 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1373359 T4624 oasc.Overseer.start Overseer (id=93615144876572675-127.0.0.1:25782_nz_fss%2Fla-n_0000000000) starting
   [junit4]   2> 1373361 T4624 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1373364 T4624 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 1373365 T4652 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1373365 T4651 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1373365 T4624 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1373369 T4624 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores
   [junit4]   2> 1373371 T4624 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores/collection1/, transient=false, config=solrconfig.xml, coreNodeName=, loadOnStartup=true, shard=, dataDir=data/, collection=control_collection, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores/collection1, schema=schema.xml, name=collection1}
   [junit4]   2> 1373371 T4624 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores/collection1/
   [junit4]   2> 1373371 T4624 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1373372 T4654 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1373372 T4654 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1373373 T4654 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1373373 T4650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1373373 T4651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:25782_nz_fss%2Fla",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25782/nz_fss/la",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state"} current state version: 0
   [junit4]   2> 1373374 T4651 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:25782_nz_fss%2Fla",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25782/nz_fss/la",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1373374 T4651 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 1373374 T4651 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1373375 T4650 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1374374 T4654 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1374375 T4654 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1374375 T4654 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1374375 T4654 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1374376 T4654 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1374376 T4654 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores/collection1/'
   [junit4]   2> 1374390 T4654 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1374392 T4654 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1374400 T4654 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1374419 T4654 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1374457 T4654 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1374559 T4654 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1374567 T4654 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1374890 T4654 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1374903 T4654 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1374905 T4654 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1374912 T4654 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1374915 T4654 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1374919 T4654 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1374919 T4654 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1374919 T4654 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1374919 T4654 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1374920 T4654 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1374920 T4654 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1374920 T4654 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1374920 T4654 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1374921 T4654 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores/collection1/data/
   [junit4]   2> 1374921 T4654 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6ba9dace
   [junit4]   2> 1374921 T4654 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores/collection1/data
   [junit4]   2> 1374922 T4654 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores/collection1/data/index/
   [junit4]   2> 1374922 T4654 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1374922 T4654 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/control-001/cores/collection1/data/index
   [junit4]   2> 1374922 T4654 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=44, maxMergeAtOnceExplicit=46, maxMergedSegmentMB=23.0185546875, floorSegmentMB=1.162109375, forceMergeDeletesPctAllowed=11.77904699044102, segmentsPerTier=31.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1374923 T4654 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@763fe6fe lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5d566a58),segFN=segments_1,generation=1}
   [junit4]   2> 1374923 T4654 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1374927 T4654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1374927 T4654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1374927 T4654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1374928 T4654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1374928 T4654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1374928 T4654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1374928 T4654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1374928 T4654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1374929 T4654 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1374951 T4654 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1374953 T4654 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1374954 T4654 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1374956 T4654 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1374960 T4654 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/plugins,/admin/luke,/admin/segments,/admin/file,/admin/threads,/admin/ping,/update,/admin/mbeans,/admin/logging,/admin/properties,/update/csv,/config,/replication,/get,/schema,standard,/update/json,/admin/system,/update/json/docs
   [junit4]   2> 1374962 T4654 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1374963 T4654 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1374963 T4654 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1374965 T4654 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1374965 T4654 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1374965 T4654 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=43, maxMergeAtOnceExplicit=46, maxMergedSegmentMB=69.14453125, floorSegmentMB=1.6064453125, forceMergeDeletesPctAllowed=4.444172208952494, segmentsPerTier=45.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4601051453371784
   [junit4]   2> 1374966 T4654 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@763fe6fe lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5d566a58),segFN=segments_1,generation=1}
   [junit4]   2> 1374966 T4654 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1374966 T4654 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@1ea1aa6a[collection1] main
   [junit4]   2> 1374966 T4654 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1374967 T4654 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1374967 T4654 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1374967 T4654 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1374968 T4654 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1374968 T4654 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1374968 T4654 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1374968 T4654 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1374968 T4654 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1374969 T4654 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1374969 T4654 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1374969 T4655 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1ea1aa6a[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1374970 T4658 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:25782/nz_fss/la collection:control_collection shard:shard1
   [junit4]   2> 1374970 T4624 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 1374970 T4624 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1374970 T4658 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1374972 T4624 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1374973 T4624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1374974 T4658 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1374974 T4662 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5aedef0c name:ZooKeeperConnection Watcher:127.0.0.1:25779/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1374974 T4624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1374975 T4624 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1374975 T4624 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1374975 T4650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1374976 T4658 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1374976 T4658 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C871 name=collection1 org.apache.solr.core.SolrCore@7aad97f2 url=http://127.0.0.1:25782/nz_fss/la/collection1 node=127.0.0.1:25782_nz_fss%2Fla C871_STATE=coll:control_collection core:collection1 props:{core=collection1, node_name=127.0.0.1:25782_nz_fss%2Fla, base_url=http://127.0.0.1:25782/nz_fss/la, state=down}
   [junit4]   2> 1374977 T4658 C:control_collection S:shard1 C871 P25782 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:25782/nz_fss/la/collection1/
   [junit4]   2> 1374977 T4651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 1374977 T4658 C:control_collection S:shard1 C871 P25782 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1374977 T4658 C:control_collection S:shard1 C871 P25782 oasc.SyncStrategy.syncToMe http://127.0.0.1:25782/nz_fss/la/collection1/ has no replicas
   [junit4]   2> 1374977 T4658 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:25782/nz_fss/la/collection1/ shard1
   [junit4]   2> 1374977 T4658 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1374978 T4650 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1374978 T4662 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1374978 T4624 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1374979 T4624 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 1374980 T4624 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1374980 T4624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1374981 T4650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1374982 T4665 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1cde29fe name:ZooKeeperConnection Watcher:127.0.0.1:25779/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1374982 T4651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25782/nz_fss/la",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 1374982 T4624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1374982 T4624 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1374985 T4650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1374986 T4651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 1374986 T4651 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 1374987 T4662 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1374987 T4650 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1375032 T4658 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:25782/nz_fss/la/collection1/ and leader is http://127.0.0.1:25782/nz_fss/la/collection1/
   [junit4]   2> 1375033 T4658 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:25782/nz_fss/la
   [junit4]   2> 1375033 T4658 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1375033 T4658 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1375033 T4658 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1375034 T4650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1375034 T4651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:25782_nz_fss%2Fla",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25782/nz_fss/la",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state"} current state version: 3
   [junit4]   2> 1375035 T4651 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:25782_nz_fss%2Fla",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25782/nz_fss/la",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1375035 T4651 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 1375036 T4650 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1375036 T4662 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1375139 T4662 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1375139 T4650 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1376129 T4624 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores/collection1
   [junit4]   2> 1376131 T4624 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001
   [junit4]   2> 1376132 T4624 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1376134 T4624 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:25795
   [junit4]   2> 1376135 T4624 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/nz_fss/la, solrconfig=solrconfig.xml, hostPort=25794, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/tempDir-001/jetty1}
   [junit4]   2> 1376135 T4624 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 1376135 T4624 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/'
   [junit4]   2> 1376167 T4624 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/solr.xml
   [junit4]   2> 1376203 T4624 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores
   [junit4]   2> 1376203 T4624 oasc.CoreContainer.<init> New CoreContainer 1379001132
   [junit4]   2> 1376203 T4624 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/]
   [junit4]   2> 1376204 T4624 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/lib
   [junit4]   2> 1376204 T4624 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/lib).
   [junit4]   2> 1376214 T4624 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1376214 T4624 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1376214 T4624 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1376215 T4624 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1376215 T4624 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1376215 T4624 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1376215 T4624 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1376215 T4624 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1376216 T4624 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1376216 T4624 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1376216 T4624 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1376221 T4624 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1376222 T4624 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1376222 T4624 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1376222 T4624 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1376222 T4624 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:25779/solr
   [junit4]   2> 1376222 T4624 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1376223 T4624 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1376223 T4624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1376225 T4677 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5ded99f0 name:ZooKeeperConnection Watcher:127.0.0.1:25779 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1376226 T4624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1376226 T4624 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1376229 T4624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1376230 T4680 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@48297555 name:ZooKeeperConnection Watcher:127.0.0.1:25779/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1376230 T4624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1376243 T4624 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1377247 T4624 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:25794_nz_fss%2Fla
   [junit4]   2> 1377248 T4624 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:25794_nz_fss%2Fla
   [junit4]   2> 1377251 T4624 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1377253 T4624 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores
   [junit4]   2> 1377255 T4624 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {schema=schema.xml, dataDir=data/, config=solrconfig.xml, name=collection1, collection=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores/collection1/, loadOnStartup=true, coreNodeName=, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores/collection1, shard=}
   [junit4]   2> 1377255 T4624 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores/collection1/
   [junit4]   2> 1377255 T4624 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1377257 T4681 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1377257 T4681 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1377258 T4650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1377258 T4681 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 1377259 T4681 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 1377259 T4651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:25794_nz_fss%2Fla",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25794/nz_fss/la",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state"} current state version: 5
   [junit4]   2> 1377259 T4681 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 1377259 T4651 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:25794_nz_fss%2Fla",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25794/nz_fss/la",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1377259 T4651 oasco.ReplicaMutator.updateState Collection already exists with numShards=3
   [junit4]   2> 1377259 T4651 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1377260 T4681 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 1377260 T4681 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1377260 T4651 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 1377261 T4680 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1377261 T4680 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 1378261 T4681 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1378262 T4681 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1378262 T4681 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1378262 T4681 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1378263 T4681 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1378263 T4681 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores/collection1/'
   [junit4]   2> 1378277 T4681 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1378279 T4681 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1378288 T4681 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1378306 T4681 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1378344 T4681 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1378446 T4681 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1378454 T4681 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1378741 T4681 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1378755 T4681 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1378757 T4681 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1378763 T4681 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1378766 T4681 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1378769 T4681 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1378770 T4681 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1378770 T4681 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1378770 T4681 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1378771 T4681 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1378771 T4681 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1378771 T4681 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1378771 T4681 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1378771 T4681 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 1378772 T4681 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6ba9dace
   [junit4]   2> 1378772 T4681 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores/collection1/data
   [junit4]   2> 1378772 T4681 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 1378773 T4681 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1378773 T4681 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 1378773 T4681 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=44, maxMergeAtOnceExplicit=46, maxMergedSegmentMB=23.0185546875, floorSegmentMB=1.162109375, forceMergeDeletesPctAllowed=11.77904699044102, segmentsPerTier=31.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1378774 T4681 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@56774af0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@31d8fee9),segFN=segments_1,generation=1}
   [junit4]   2> 1378774 T4681 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1378778 T4681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1378779 T4681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1378779 T4681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1378779 T4681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1378779 T4681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1378779 T4681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1378780 T4681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1378780 T4681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1378780 T4681 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1378797 T4681 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1378799 T4681 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1378801 T4681 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1378802 T4681 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1378807 T4681 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/plugins,/admin/luke,/admin/segments,/admin/file,/admin/threads,/admin/ping,/update,/admin/mbeans,/admin/logging,/admin/properties,/update/csv,/config,/replication,/get,/schema,standard,/update/json,/admin/system,/update/json/docs
   [junit4]   2> 1378809 T4681 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1378810 T4681 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1378811 T4681 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1378812 T4681 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1378812 T4681 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1378813 T4681 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=43, maxMergeAtOnceExplicit=46, maxMergedSegmentMB=69.14453125, floorSegmentMB=1.6064453125, forceMergeDeletesPctAllowed=4.444172208952494, segmentsPerTier=45.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4601051453371784
   [junit4]   2> 1378813 T4681 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@56774af0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@31d8fee9),segFN=segments_1,generation=1}
   [junit4]   2> 1378813 T4681 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1378814 T4681 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@7a6a994e[collection1] main
   [junit4]   2> 1378814 T4681 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1378814 T4681 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1378814 T4681 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1378815 T4681 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1378815 T4681 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1378815 T4681 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1378815 T4681 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1378816 T4681 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1378816 T4681 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1378816 T4681 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1378817 T4682 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7a6a994e[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1378817 T4681 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1378818 T4685 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:25794/nz_fss/la collection:collection1 shard:shard1
   [junit4]   2> 1378818 T4624 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 1378818 T4624 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1378818 T4685 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1378821 T4685 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1378822 T4650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1378823 T4685 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1378823 T4685 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C872 name=collection1 org.apache.solr.core.SolrCore@1bffbe5b url=http://127.0.0.1:25794/nz_fss/la/collection1 node=127.0.0.1:25794_nz_fss%2Fla C872_STATE=coll:collection1 core:collection1 props:{core=collection1, node_name=127.0.0.1:25794_nz_fss%2Fla, base_url=http://127.0.0.1:25794/nz_fss/la, state=down}
   [junit4]   2> 1378823 T4685 C:collection1 S:shard1 C872 P25794 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:25794/nz_fss/la/collection1/
   [junit4]   2> 1378823 T4651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 1378823 T4685 C:collection1 S:shard1 C872 P25794 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1378824 T4685 C:collection1 S:shard1 C872 P25794 oasc.SyncStrategy.syncToMe http://127.0.0.1:25794/nz_fss/la/collection1/ has no replicas
   [junit4]   2> 1378824 T4685 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:25794/nz_fss/la/collection1/ shard1
   [junit4]   2> 1378824 T4685 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1378824 T4651 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 1378825 T4680 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1378825 T4680 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 1378827 T4651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25794/nz_fss/la",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 1378827 T4651 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 1378828 T4680 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1378828 T4680 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 1378829 T4650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1378878 T4685 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:25794/nz_fss/la/collection1/ and leader is http://127.0.0.1:25794/nz_fss/la/collection1/
   [junit4]   2> 1378879 T4685 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:25794/nz_fss/la
   [junit4]   2> 1378879 T4685 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1378879 T4685 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 1378879 T4685 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1378880 T4650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1378880 T4651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:25794_nz_fss%2Fla",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25794/nz_fss/la",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state"} current state version: 5
   [junit4]   2> 1378881 T4651 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:25794_nz_fss%2Fla",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25794/nz_fss/la",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1378881 T4685 C:collection1 S:shard1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 1378882 T4651 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 3
   [junit4]   2> 1378882 T4680 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1378883 T4680 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 4 
   [junit4]   2> 1379746 T4624 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-2-001/cores/collection1
   [junit4]   2> 1379748 T4624 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-2-001
   [junit4]   2> 1379749 T4624 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1379752 T4624 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:25804
   [junit4]   2> 1379752 T4624 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-2-001/cores, hostPort=25803, hostContext=/nz_fss/la}
   [junit4]   2> 1379753 T4624 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 1379753 T4624 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-2-001/'
   [junit4]   2> 1379782 T4624 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-2-001/solr.xml
   [junit4]   2> 1379816 T4624 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-2-001/cores
   [junit4]   2> 1379817 T4624 oasc.CoreContainer.<init> New CoreContainer 1199359807
   [junit4]   2> 1379817 T4624 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-2-001/]
   [junit4]   2> 1379817 T4624 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-2-001/lib
   [junit4]   2> 1379818 T4624 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-2-001/lib).
   [junit4]   2> 1379827 T4624 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1379827 T4624 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1379827 T4624 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1379827 T4624 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1379828 T4624 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1379828 T4624 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1379828 T4624 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1379828 T4624 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1379828 T4624 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1379829 T4624 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1379829 T4624 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1379833 T4624 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1379834 T4624 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1379834 T4624 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1379834 T4624 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1379835 T4624 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:25779/solr
   [junit4]   2> 1379835 T4624 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1379835 T4624 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1379836 T4624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1379837 T4698 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3ca4f3b4 name:ZooKeeperConnection Watcher:127.0.0.1:25779 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1379838 T4624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1379838 T4624 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1379840 T4624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1379841 T4701 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4e7c6687 name:ZooKeeperConnection Watcher:127.0.0.1:25779/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1379841 T4624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1379845 T4624 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1380848 T4624 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:25803_nz_fss%2Fla
   [junit4]   2> 1380849 T4624 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:25803_nz_fss%2Fla
   [junit4]   2> 1380852 T4624 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1380855 T4624 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-2-001/cores
   [junit4]   2> 1380857 T4624 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {coreNodeName=, collection=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-2-001/cores/collection1/, dataDir=data/, name=collection1, shard=, schema=schema.xml, transient=false, loadOnStartup=true, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-2-001/cores/collection1, config=solrconfig.xml}
   [junit4]   2> 1380857 T4624 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001/shard-2-001/cores/collection1/
   [junit4]   2> 1380858 T4624 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1380859 T4702 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1380859 T4702 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1380860 T4650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1380861 T4702 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 1380861 T4702 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 1380861 T4651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:25803_nz_fss%2Fla",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25803/nz_fss/la",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state"} current state version: 5
   [junit4]   2> 1380861 T4702 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 1380862 T4651 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:25803_nz_fss%2Fla",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25803/nz_fss/la",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 1380862 T4651 oasco.ReplicaMutator.updateState Collection already exists with numShards=3
   [junit4]   2> 1380862 T4651 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 1380862 T4702 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 4 
   [junit4]   2> 1380862 T4702 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1380863 T4651 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 4
   [junit4]   2> 1380863 T4701 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1380863 T4680 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1380864 T4701 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 5 
   [junit4]   2> 1380864 T4680 oascc.ZkStateReader.update

[...truncated too long message...]

.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:278)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #4: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=4829, name=updateExecutor-653-thread-6, state=RUNNABLE, group=TGRP-ReplicationFactorTest]
   [junit4]    > Caused by: java.util.concurrent.RejectedExecutionException: Task org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner@282848d9 rejected from java.util.concurrent.ThreadPoolExecutor@e71bb73[Shutting down, pool size = 4, active threads = 4, queued tasks = 0, completed tasks = 46]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([BFFCE504F86C54DD]:0)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:278)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #5: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=4832, name=updateExecutor-653-thread-7, state=RUNNABLE, group=TGRP-ReplicationFactorTest]
   [junit4]    > Caused by: java.util.concurrent.RejectedExecutionException: Task org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner@5b761ce3 rejected from java.util.concurrent.ThreadPoolExecutor@e71bb73[Shutting down, pool size = 3, active threads = 3, queued tasks = 0, completed tasks = 47]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([BFFCE504F86C54DD]:0)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:278)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #6: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=4835, name=updateExecutor-653-thread-8, state=RUNNABLE, group=TGRP-ReplicationFactorTest]
   [junit4]    > Caused by: java.util.concurrent.RejectedExecutionException: Task org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner@f2d440b rejected from java.util.concurrent.ThreadPoolExecutor@e71bb73[Shutting down, pool size = 2, active threads = 2, queued tasks = 0, completed tasks = 48]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([BFFCE504F86C54DD]:0)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:278)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #7: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=4822, name=updateExecutor-653-thread-4, state=RUNNABLE, group=TGRP-ReplicationFactorTest]
   [junit4]    > Caused by: java.util.concurrent.RejectedExecutionException: Task org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner@ad15bd2 rejected from java.util.concurrent.ThreadPoolExecutor@e71bb73[Shutting down, pool size = 1, active threads = 1, queued tasks = 0, completed tasks = 49]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([BFFCE504F86C54DD]:0)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:278)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1955257 T4624 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BFFCE504F86C54DD-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {_version_=PostingsFormat(name=Asserting), timestamp=PostingsFormat(name=LuceneFixedGap), id=PostingsFormat(name=Memory doPackFST= true), a_t=PostingsFormat(name=LuceneFixedGap), intDefault=PostingsFormat(name=Asserting), range_facet_l=PostingsFormat(name=LuceneFixedGap), text=PostingsFormat(name=LuceneVarGapDocFreqInterval), multiDefault=PostingsFormat(name=LuceneFixedGap)}, docValues:{timestamp=DocValuesFormat(name=Asserting)}, sim=DefaultSimilarity, locale=ar_OM, timezone=Europe/Tiraspol
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=78560712,total=365953024
   [junit4]   2> NOTE: All tests run in this JVM: [TestSystemIdResolver, TestSchemaSimilarityResource, PolyFieldTest, CoreAdminCreateDiscoverTest, PrimitiveFieldTypeTest, RegexBoostProcessorTest, PreAnalyzedUpdateProcessorTest, HttpPartitionTest, PreAnalyzedFieldTest, TestSolrQueryParserResource, IndexSchemaRuntimeFieldTest, TestArbitraryIndexDir, TestReload, NumericFieldsTest, TestJmxMonitoredMap, TestBulkSchemaAPI, TestManagedResource, TestSolrQueryParserDefaultOperatorResource, TestFieldTypeResource, AnalysisAfterCoreReloadTest, DocValuesMissingTest, DistributedMLTComponentTest, EchoParamsTest, TermVectorComponentDistributedTest, TestCloudInspectUtil, QueryResultKeyTest, TestDistributedGrouping, HdfsBasicDistributedZkTest, AddSchemaFieldsUpdateProcessorFactoryTest, TestCollationField, TestLRUCache, TestSolrDeletionPolicy1, TestExtendedDismaxParser, TestTrackingShardHandlerFactory, SolrInfoMBeanTest, BlockDirectoryTest, XsltUpdateRequestHandlerTest, UnloadDistributedZkTest, TestBinaryField, DirectUpdateHandlerTest, CollectionReloadTest, SimpleMLTQParserTest, TestUniqueKeyFieldResource, TestDynamicLoading, TestLeaderElectionZkExpiry, TestManagedSchemaDynamicFieldResource, TestAddFieldRealTimeGet, TestDistribDocBasedVersion, DistributedIntervalFacetingTest, TestJsonFacets, UpdateParamsTest, HdfsSyncSliceTest, TestFastLRUCache, IndexSchemaTest, FieldAnalysisRequestHandlerTest, TestManagedSchemaFieldTypeResource, TestConfig, TestSolrJ, CollectionsAPIDistributedZkTest, BasicDistributedZkTest, MultiThreadedOCPTest, ReplicationFactorTest]
   [junit4] Completed on J0 in 583.73s, 1 test, 1 error <<< FAILURES!

[...truncated 614 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 480 suites, 1905 tests, 1 error, 61 ignored (21 assumptions)

Total time: 60 minutes 33 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 218537067 bytes
Compression is 0.0%
Took 41 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2907 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2907/

1 tests failed.
FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
java.net.SocketTimeoutException: Read timed out

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: java.net.SocketTimeoutException: Read timed out
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.sendDocsWithRetry(AbstractFullDistribZkTestBase.java:774)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:277)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9900 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/init-core-data-001
   [junit4]   2> 440254 T1875 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 440258 T1875 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 440258 T1876 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 440358 T1875 oasc.ZkTestServer.run start zk server on port:14242
   [junit4]   2> 440359 T1875 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 440360 T1875 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 440362 T1883 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@29b8b880 name:ZooKeeperConnection Watcher:127.0.0.1:14242 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 440363 T1875 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 440363 T1875 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 440363 T1875 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 440365 T1875 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 440366 T1875 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 440367 T1886 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3bb633cd name:ZooKeeperConnection Watcher:127.0.0.1:14242/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 440367 T1875 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 440367 T1875 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 440367 T1875 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 440368 T1875 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 440369 T1875 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 440370 T1875 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 440372 T1875 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 440372 T1875 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 440374 T1875 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 440374 T1875 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 440475 T1875 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 440476 T1875 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 440477 T1875 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 440477 T1875 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 440479 T1875 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 440479 T1875 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 440480 T1875 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 440481 T1875 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 440482 T1875 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 440482 T1875 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 440483 T1875 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 440484 T1875 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 440485 T1875 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 440485 T1875 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 440486 T1875 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 440487 T1875 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 440488 T1875 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 440488 T1875 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 441510 T1875 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores/collection1
   [junit4]   2> 441513 T1875 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 441517 T1875 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:14246
   [junit4]   2> 441517 T1875 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=14245, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores}
   [junit4]   2> 441518 T1875 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 441518 T1875 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/'
   [junit4]   2> 441547 T1875 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/solr.xml
   [junit4]   2> 441599 T1875 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores
   [junit4]   2> 441600 T1875 oasc.CoreContainer.<init> New CoreContainer 1380014511
   [junit4]   2> 441600 T1875 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/]
   [junit4]   2> 441601 T1875 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/lib
   [junit4]   2> 441602 T1875 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/lib).
   [junit4]   2> 441611 T1875 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 441611 T1875 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 441612 T1875 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 441612 T1875 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 441612 T1875 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 441612 T1875 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 441612 T1875 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 441613 T1875 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 441613 T1875 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 441613 T1875 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 441613 T1875 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 441618 T1875 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 441619 T1875 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 441619 T1875 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 441619 T1875 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 441619 T1875 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:14242/solr
   [junit4]   2> 441619 T1875 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 441620 T1875 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 441620 T1875 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 441622 T1898 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@994d153 name:ZooKeeperConnection Watcher:127.0.0.1:14242 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 441623 T1875 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 441623 T1875 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 441624 T1875 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 441625 T1901 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5fcb98d6 name:ZooKeeperConnection Watcher:127.0.0.1:14242/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 441626 T1875 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 441627 T1875 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 441628 T1875 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 441629 T1875 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 441631 T1875 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 441632 T1875 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 441633 T1875 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 441634 T1875 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 441635 T1875 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 441636 T1875 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:14245_
   [junit4]   2> 441636 T1875 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:14245_
   [junit4]   2> 441637 T1875 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 441638 T1875 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 441639 T1875 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 441640 T1875 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:14245_
   [junit4]   2> 441640 T1875 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 441641 T1875 oasc.Overseer.start Overseer (id=93614406352961539-127.0.0.1:14245_-n_0000000000) starting
   [junit4]   2> 441642 T1875 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 441646 T1875 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 441647 T1903 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 441647 T1902 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 441647 T1875 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 441650 T1875 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores
   [junit4]   2> 441651 T1875 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {shard=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores/collection1, loadOnStartup=true, dataDir=data/, transient=false, coreNodeName=, schema=schema.xml, config=solrconfig.xml, collection=control_collection, name=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores/collection1/}
   [junit4]   2> 441652 T1875 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores/collection1/
   [junit4]   2> 441652 T1875 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 441653 T1905 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 441653 T1905 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 441653 T1905 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 441653 T1901 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 441654 T1902 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14245",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:14245_",
   [junit4]   2> 	  "shard":null} current state version: 0
   [junit4]   2> 441655 T1902 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14245",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:14245_",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 441655 T1902 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 441655 T1902 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 441656 T1901 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 442654 T1905 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 442655 T1905 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 442655 T1905 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 442656 T1905 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 442656 T1905 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 442656 T1905 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores/collection1/'
   [junit4]   2> 442674 T1905 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 442676 T1905 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 442686 T1905 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 442709 T1905 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 442761 T1905 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 442862 T1905 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 442873 T1905 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 443284 T1905 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 443301 T1905 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 443304 T1905 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 443314 T1905 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 443318 T1905 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 443322 T1905 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 443323 T1905 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 443323 T1905 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 443323 T1905 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 443324 T1905 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 443324 T1905 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 443325 T1905 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 443325 T1905 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 443325 T1905 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores/collection1/data/
   [junit4]   2> 443326 T1905 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@42d7ecc1
   [junit4]   2> 443326 T1905 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores/collection1/data
   [junit4]   2> 443327 T1905 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores/collection1/data/index/
   [junit4]   2> 443327 T1905 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 443327 T1905 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/control-001/cores/collection1/data/index
   [junit4]   2> 443328 T1905 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=42, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6542973963737145]
   [junit4]   2> 443329 T1905 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@45455d46 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4ceec231),segFN=segments_1,generation=1}
   [junit4]   2> 443329 T1905 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 443334 T1905 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 443335 T1905 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 443335 T1905 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 443335 T1905 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 443336 T1905 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 443336 T1905 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 443336 T1905 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 443337 T1905 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 443337 T1905 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 443357 T1905 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 443360 T1905 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 443362 T1905 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 443364 T1905 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 443369 T1905 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/file,/admin/ping,/admin/luke,/admin/threads,/admin/properties,/admin/segments,/get,/admin/mbeans,/schema,/update/json/docs,/config,/update/csv,/admin/logging,/admin/plugins,/update,/update/json,/replication,/admin/system,standard
   [junit4]   2> 443371 T1905 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 443373 T1905 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 443373 T1905 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 443374 T1905 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 443375 T1905 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 443375 T1905 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=37, maxMergedSegmentMB=57.2275390625, floorSegmentMB=0.2529296875, forceMergeDeletesPctAllowed=20.90666064053401, segmentsPerTier=22.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 443376 T1905 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@45455d46 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4ceec231),segFN=segments_1,generation=1}
   [junit4]   2> 443376 T1905 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 443377 T1905 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@73d0e76e[collection1] main
   [junit4]   2> 443377 T1905 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 443377 T1905 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 443378 T1905 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 443378 T1905 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 443378 T1905 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 443378 T1905 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 443379 T1905 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 443379 T1905 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 443379 T1905 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 443380 T1905 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 443380 T1905 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 443381 T1906 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@73d0e76e[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 443381 T1909 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:14245 collection:control_collection shard:shard1
   [junit4]   2> 443381 T1875 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 443382 T1875 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 443382 T1909 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 443385 T1909 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 443386 T1901 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 443386 T1909 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 443386 T1909 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 443386 T1875 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 443386 T1902 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C206 name=collection1 org.apache.solr.core.SolrCore@1ab3596b url=http://127.0.0.1:14245/collection1 node=127.0.0.1:14245_ C206_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:14245, core=collection1, node_name=127.0.0.1:14245_}
   [junit4]   2> 443386 T1909 C:control_collection S:shard1 C206 P14245 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:14245/collection1/
   [junit4]   2> 443387 T1875 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 443387 T1909 C:control_collection S:shard1 C206 P14245 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 443388 T1909 C:control_collection S:shard1 C206 P14245 oasc.SyncStrategy.syncToMe http://127.0.0.1:14245/collection1/ has no replicas
   [junit4]   2> 443388 T1901 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 443388 T1909 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:14245/collection1/ shard1
   [junit4]   2> 443388 T1909 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 443389 T1913 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7f02e185 name:ZooKeeperConnection Watcher:127.0.0.1:14242/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 443389 T1875 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 443389 T1875 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 443390 T1875 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 443391 T1901 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 443392 T1902 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14245",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 443392 T1875 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 443494 T1913 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 443494 T1901 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 443544 T1909 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:14245/collection1/ and leader is http://127.0.0.1:14245/collection1/
   [junit4]   2> 443545 T1909 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:14245
   [junit4]   2> 443545 T1909 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 443545 T1909 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 443545 T1909 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 443546 T1901 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 443546 T1902 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14245",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:14245_",
   [junit4]   2> 	  "shard":"shard1"} current state version: 3
   [junit4]   2> 443547 T1902 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14245",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:14245_",
   [junit4]   2> 	  "shard":"shard1"}
   [junit4]   2> 443650 T1901 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 443650 T1913 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 444808 T1875 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores/collection1
   [junit4]   2> 444811 T1875 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001
   [junit4]   2> 444812 T1875 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 444815 T1875 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:14251
   [junit4]   2> 444815 T1875 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostPort=14250}
   [junit4]   2> 444816 T1875 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 444816 T1875 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/'
   [junit4]   2> 444856 T1875 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/solr.xml
   [junit4]   2> 444896 T1875 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores
   [junit4]   2> 444896 T1875 oasc.CoreContainer.<init> New CoreContainer 1555043384
   [junit4]   2> 444897 T1875 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/]
   [junit4]   2> 444897 T1875 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/lib
   [junit4]   2> 444897 T1875 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/lib).
   [junit4]   2> 444906 T1875 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 444907 T1875 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 444907 T1875 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 444907 T1875 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 444907 T1875 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 444908 T1875 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 444908 T1875 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 444908 T1875 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 444908 T1875 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 444908 T1875 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 444909 T1875 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 444914 T1875 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 444915 T1875 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 444915 T1875 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 444915 T1875 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 444915 T1875 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:14242/solr
   [junit4]   2> 444916 T1875 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 444916 T1875 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 444917 T1875 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 444918 T1925 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3fa3586c name:ZooKeeperConnection Watcher:127.0.0.1:14242 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 444919 T1875 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 444920 T1875 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 444921 T1875 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 444922 T1928 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@128b39c7 name:ZooKeeperConnection Watcher:127.0.0.1:14242/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 444922 T1875 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 444925 T1875 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 445928 T1875 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:14250_
   [junit4]   2> 445929 T1875 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:14250_
   [junit4]   2> 445932 T1875 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 445934 T1875 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores
   [junit4]   2> 445935 T1875 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {transient=false, shard=, coreNodeName=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores/collection1/, config=solrconfig.xml, dataDir=data/, collection=collection1, loadOnStartup=true, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores/collection1, name=collection1, schema=schema.xml}
   [junit4]   2> 445936 T1875 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores/collection1/
   [junit4]   2> 445936 T1875 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 445937 T1929 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 445937 T1929 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 445937 T1929 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 445937 T1901 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 445938 T1902 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14250",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:14250_",
   [junit4]   2> 	  "shard":null} current state version: 4
   [junit4]   2> 445938 T1902 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14250",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:14250_",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 445939 T1902 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 445939 T1902 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard3
   [junit4]   2> 445940 T1901 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 445940 T1928 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 445940 T1913 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 446938 T1929 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 446939 T1929 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 446939 T1929 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 446939 T1929 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 446940 T1929 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 446940 T1929 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores/collection1/'
   [junit4]   2> 446954 T1929 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 446956 T1929 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 446965 T1929 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 446982 T1929 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 447020 T1929 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 447121 T1929 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 447129 T1929 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 447470 T1929 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 447482 T1929 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 447484 T1929 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 447492 T1929 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 447495 T1929 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 447498 T1929 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 447498 T1929 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 447498 T1929 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 447498 T1929 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 447499 T1929 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 447499 T1929 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 447499 T1929 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 447499 T1929 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 447500 T1929 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 447500 T1929 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@42d7ecc1
   [junit4]   2> 447501 T1929 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores/collection1/data
   [junit4]   2> 447501 T1929 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 447501 T1929 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 447501 T1929 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 447501 T1929 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=42, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6542973963737145]
   [junit4]   2> 447502 T1929 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@a1b712e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@fa38831),segFN=segments_1,generation=1}
   [junit4]   2> 447502 T1929 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 447507 T1929 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 447507 T1929 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 447507 T1929 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 447508 T1929 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 447508 T1929 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 447508 T1929 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 447508 T1929 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 447509 T1929 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 447509 T1929 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 447523 T1929 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 447524 T1929 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 447526 T1929 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 447528 T1929 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 447533 T1929 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/file,/admin/ping,/admin/luke,/admin/threads,/admin/properties,/admin/segments,/get,/admin/mbeans,/schema,/update/json/docs,/config,/update/csv,/admin/logging,/admin/plugins,/update,/update/json,/replication,/admin/system,standard
   [junit4]   2> 447535 T1929 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 447536 T1929 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 447536 T1929 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 447537 T1929 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 447537 T1929 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 447539 T1929 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=37, maxMergedSegmentMB=57.2275390625, floorSegmentMB=0.2529296875, forceMergeDeletesPctAllowed=20.90666064053401, segmentsPerTier=22.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 447539 T1929 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@a1b712e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@fa38831),segFN=segments_1,generation=1}
   [junit4]   2> 447539 T1929 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 447540 T1929 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@57b843f7[collection1] main
   [junit4]   2> 447540 T1929 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 447540 T1929 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 447541 T1929 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 447541 T1929 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 447541 T1929 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 447541 T1929 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 447542 T1929 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 447542 T1929 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 447542 T1929 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 447542 T1929 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 447543 T1930 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@57b843f7[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 447543 T1929 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 447544 T1933 C:collection1 S:shard3 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:14250 collection:collection1 shard:shard3
   [junit4]   2> 447544 T1875 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 447544 T1875 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 447545 T1933 C:collection1 S:shard3 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard3/election
   [junit4]   2> 447548 T1933 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard3
   [junit4]   2> 447549 T1901 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 447549 T1933 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 447549 T1933 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C207 name=collection1 org.apache.solr.core.SolrCore@ab3b34e url=http://127.0.0.1:14250/collection1 node=127.0.0.1:14250_ C207_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:14250, core=collection1, node_name=127.0.0.1:14250_}
   [junit4]   2> 447549 T1933 C:collection1 S:shard3 C207 P14250 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:14250/collection1/
   [junit4]   2> 447549 T1902 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 447549 T1933 C:collection1 S:shard3 C207 P14250 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 447549 T1933 C:collection1 S:shard3 C207 P14250 oasc.SyncStrategy.syncToMe http://127.0.0.1:14250/collection1/ has no replicas
   [junit4]   2> 447550 T1933 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:14250/collection1/ shard3
   [junit4]   2> 447550 T1933 C:collection1 S:shard3 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard3
   [junit4]   2> 447550 T1901 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 447550 T1928 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 447550 T1913 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 447553 T1901 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 447553 T1902 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14250",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 447655 T1901 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 447655 T1913 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 447655 T1928 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 447705 T1933 C:collection1 S:shard3 oasc.ZkController.register We are http://127.0.0.1:14250/collection1/ and leader is http://127.0.0.1:14250/collection1/
   [junit4]   2> 447706 T1933 C:collection1 S:shard3 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:14250
   [junit4]   2> 447706 T1933 C:collection1 S:shard3 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 447706 T1933 C:collection1 S:shard3 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 447706 T1933 C:collection1 S:shard3 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 447707 T1901 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 447708 T1902 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14250",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:14250_",
   [junit4]   2> 	  "shard":"shard3"} current state version: 7
   [junit4]   2> 447709 T1902 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14250",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:14250_",
   [junit4]   2> 	  "shard":"shard3"}
   [junit4]   2> 447811 T1928 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 447811 T1901 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 447811 T1913 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 448664 T1875 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001/cores/collection1
   [junit4]   2> 448666 T1875 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001
   [junit4]   2> 448668 T1875 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 448670 T1875 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:14256
   [junit4]   2> 448670 T1875 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, hostPort=14255, solrconfig=solrconfig.xml, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/tempDir-001/jetty2}
   [junit4]   2> 448671 T1875 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 448671 T1875 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001/'
   [junit4]   2> 448720 T1875 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001/solr.xml
   [junit4]   2> 448781 T1875 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001/cores
   [junit4]   2> 448781 T1875 oasc.CoreContainer.<init> New CoreContainer 2041762435
   [junit4]   2> 448782 T1875 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001/]
   [junit4]   2> 448782 T1875 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001/lib
   [junit4]   2> 448783 T1875 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001/lib).
   [junit4]   2> 448797 T1875 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 448797 T1875 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 448798 T1875 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 448798 T1875 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 448798 T1875 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 448799 T1875 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 448799 T1875 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 448799 T1875 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 448800 T1875 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 448800 T1875 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 448800 T1875 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 448807 T1875 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 448808 T1875 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 448809 T1875 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 448809 T1875 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 448809 T1875 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:14242/solr
   [junit4]   2> 448810 T1875 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 448810 T1875 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 448811 T1875 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 448813 T1946 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@657aa9f6 name:ZooKeeperConnection Watcher:127.0.0.1:14242 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 448814 T1875 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 448814 T1875 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 448816 T1875 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 448817 T1949 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@14f9a04e name:ZooKeeperConnection Watcher:127.0.0.1:14242/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 448818 T1875 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 448822 T1875 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 449825 T1875 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:14255_
   [junit4]   2> 449826 T1875 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:14255_
   [junit4]   2> 449829 T1875 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 449831 T1875 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001/cores
   [junit4]   2> 449832 T1875 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {coreNodeName=, shard=, schema=schema.xml, dataDir=data/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001/cores/collection1, name=collection1, collection=collection1, config=solrconfig.xml, loadOnStartup=true, transient=false, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001/cores/collection1/}
   [junit4]   2> 449833 T1875 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001/cores/collection1/
   [junit4]   2> 449833 T1875 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 449834 T1950 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 449834 T1950 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 449834 T1950 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 449834 T1901 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 449835 T1902 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14255",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:14255_",
   [junit4]   2> 	  "shard":null} current state version: 8
   [junit4]   2> 449836 T1902 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14255",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:14255_",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 449836 T1902 oasco.ReplicaMutator.updateState Collection already exists with numShards=3
   [junit4]   2> 449836 T1902 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 449837 T1949 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 449837 T1913 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 449837 T1901 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 449837 T1928 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 450835 T1950 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 450836 T1950 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 450836 T1950 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 450837 T1950 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 450837 T1950 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 450837 T1950 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001/shard-2-001/cores/collection1/'
   [junit4]   2> 450855 T1950 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 450857 T1950 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 450868 T1950 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 450891 T1950 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 450941 T1950 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 451042 T1950 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 451053 T1950 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 451470 T1950 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 451486 T1950 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 451489 T1950 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 451499 T1950 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 451504 T1950 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 451508 T1950 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 451509 T1950 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 451510 T1950 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 451510 T1950 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 451511 T1950 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 451511 T1950 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 451511 T1950 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 451512 T1950 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 451512 T1950 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/worksp

[...truncated too long message...]

ent.ThreadPoolExecutor@bc189bc[Shutting down, pool size = 5, active threads = 5, queued tasks = 0, completed tasks = 48]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([457F35F1A3900CAB]:0)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:278)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #4: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=2070, name=updateExecutor-278-thread-5, state=RUNNABLE, group=TGRP-ReplicationFactorTest]
   [junit4]    > Caused by: java.util.concurrent.RejectedExecutionException: Task org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner@350152a3 rejected from java.util.concurrent.ThreadPoolExecutor@bc189bc[Shutting down, pool size = 4, active threads = 4, queued tasks = 0, completed tasks = 49]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([457F35F1A3900CAB]:0)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:278)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #5: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=2073, name=updateExecutor-278-thread-6, state=RUNNABLE, group=TGRP-ReplicationFactorTest]
   [junit4]    > Caused by: java.util.concurrent.RejectedExecutionException: Task org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner@162ec62c rejected from java.util.concurrent.ThreadPoolExecutor@bc189bc[Shutting down, pool size = 3, active threads = 3, queued tasks = 0, completed tasks = 50]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([457F35F1A3900CAB]:0)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:278)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #6: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=2076, name=updateExecutor-278-thread-7, state=RUNNABLE, group=TGRP-ReplicationFactorTest]
   [junit4]    > Caused by: java.util.concurrent.RejectedExecutionException: Task org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner@743f2543 rejected from java.util.concurrent.ThreadPoolExecutor@bc189bc[Shutting down, pool size = 2, active threads = 2, queued tasks = 0, completed tasks = 51]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([457F35F1A3900CAB]:0)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:278)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #7: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=2079, name=updateExecutor-278-thread-8, state=RUNNABLE, group=TGRP-ReplicationFactorTest]
   [junit4]    > Caused by: java.util.concurrent.RejectedExecutionException: Task org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner@43cb4a37 rejected from java.util.concurrent.ThreadPoolExecutor@bc189bc[Shutting down, pool size = 1, active threads = 1, queued tasks = 0, completed tasks = 52]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([457F35F1A3900CAB]:0)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:278)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1012391 T1875 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 457F35F1A3900CAB-001
   [junit4]   2> 572143 T1874 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {multiDefault=BlockTreeOrds(blocksize=128), a_t=PostingsFormat(name=Memory doPackFST= false), text=FST50, range_facet_l=PostingsFormat(name=Memory doPackFST= false), intDefault=PostingsFormat(name=Memory doPackFST= false), timestamp=PostingsFormat(name=Memory doPackFST= false), _version_=PostingsFormat(name=Memory doPackFST= false), id=PostingsFormat(name=LuceneVarGapFixedInterval)}, docValues:{timestamp=DocValuesFormat(name=SimpleText)}, sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {}, locale=en_IE, timezone=America/Kentucky/Monticello
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=171993136,total=303038464
   [junit4]   2> NOTE: All tests run in this JVM: [TestCoreContainer, ChaosMonkeySafeLeaderTest, TestLMDirichletSimilarityFactory, TestSimpleTrackingShardHandler, AnalysisErrorHandlingTest, OpenExchangeRatesOrgProviderTest, TestSystemIdResolver, LeaderElectionTest, TestExactStatsCache, HighlighterMaxOffsetTest, SolrRequestParserTest, HdfsBasicDistributedZk2Test, CurrencyFieldXmlFileTest, TestMiniSolrCloudCluster, RAMDirectoryFactoryTest, TestCollectionAPI, PrimUtilsTest, HdfsCollectionsAPIDistributedZkTest, TestPostingsSolrHighlighter, TestSerializedLuceneMatchVersion, TestFieldTypeCollectionResource, TestJmxIntegration, TestPseudoReturnFields, TestSolrCoreProperties, TestImplicitCoreProperties, XsltUpdateRequestHandlerTest, CSVRequestHandlerTest, TestSolrJ, DateMathParserTest, TestTrie, TestRemoteStreaming, DistanceUnitsTest, TestEmbeddedSolrServerConstructors, CustomCollectionTest, ReplicationFactorTest]
   [junit4] Completed on J3 in 572.42s, 1 test, 1 error <<< FAILURES!

[...truncated 868 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 480 suites, 1905 tests, 1 error, 61 ignored (21 assumptions)

Total time: 60 minutes 12 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 204630843 bytes
Compression is 0.0%
Took 36 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2906 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2906/

2 tests failed.
REGRESSION:  org.apache.solr.handler.component.DistributedMLTComponentTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:35385//collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:35385//collection1
	at __randomizedtesting.SeedInfo.seed([2902CDAD2A4D727B:A156F27784B11F83]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:566)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:943)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:558)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:606)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:588)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:567)
	at org.apache.solr.handler.component.DistributedMLTComponentTest.test(DistributedMLTComponentTest.java:126)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 51 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
java.net.SocketTimeoutException: Read timed out

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: java.net.SocketTimeoutException: Read timed out
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.sendDocsWithRetry(AbstractFullDistribZkTestBase.java:774)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:277)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9961 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/init-core-data-001
   [junit4]   2> 677096 T2342 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 677103 T2342 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 677103 T2343 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 677203 T2342 oasc.ZkTestServer.run start zk server on port:18161
   [junit4]   2> 677204 T2342 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 677205 T2342 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 677210 T2350 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4d07fe53 name:ZooKeeperConnection Watcher:127.0.0.1:18161 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 677210 T2342 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 677211 T2342 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 677211 T2342 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 677215 T2342 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 677215 T2342 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 677217 T2353 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5a5efbc7 name:ZooKeeperConnection Watcher:127.0.0.1:18161/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 677217 T2342 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 677218 T2342 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 677218 T2342 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 677221 T2342 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 677222 T2342 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 677224 T2342 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 677226 T2342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 677227 T2342 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 677229 T2342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 677230 T2342 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 677332 T2342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 677332 T2342 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 677334 T2342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 677335 T2342 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 677336 T2342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 677337 T2342 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 677338 T2342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 677339 T2342 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 677340 T2342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 677341 T2342 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 677342 T2342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 677343 T2342 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 677344 T2342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 677345 T2342 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 677346 T2342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 677347 T2342 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 677348 T2342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 677349 T2342 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 678800 T2342 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores/collection1
   [junit4]   2> 678803 T2342 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 678805 T2342 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:56011
   [junit4]   2> 678805 T2342 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=23759, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores}
   [junit4]   2> 678806 T2342 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 678806 T2342 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/'
   [junit4]   2> 678841 T2342 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/solr.xml
   [junit4]   2> 678883 T2342 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores
   [junit4]   2> 678884 T2342 oasc.CoreContainer.<init> New CoreContainer 2012355157
   [junit4]   2> 678884 T2342 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/]
   [junit4]   2> 678884 T2342 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/lib
   [junit4]   2> 678885 T2342 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/lib).
   [junit4]   2> 678895 T2342 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 678895 T2342 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 678895 T2342 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 678896 T2342 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 678896 T2342 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 678896 T2342 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 678896 T2342 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 678896 T2342 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 678897 T2342 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 678897 T2342 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 678897 T2342 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 678902 T2342 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 678903 T2342 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 678903 T2342 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 678903 T2342 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 678904 T2342 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:18161/solr
   [junit4]   2> 678904 T2342 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 678904 T2342 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 678905 T2342 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 678906 T2365 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7c251cb1 name:ZooKeeperConnection Watcher:127.0.0.1:18161 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 678907 T2342 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 678907 T2342 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 678909 T2342 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 678910 T2368 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@69f080ec name:ZooKeeperConnection Watcher:127.0.0.1:18161/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 678911 T2342 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 678912 T2342 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 678913 T2342 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 678915 T2342 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 678916 T2342 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 678917 T2342 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 678919 T2342 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 678920 T2342 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 678921 T2342 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 678922 T2342 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:23759_
   [junit4]   2> 678922 T2342 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:23759_
   [junit4]   2> 678923 T2342 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 678924 T2342 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 678925 T2342 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 678926 T2342 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:23759_
   [junit4]   2> 678926 T2342 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 678927 T2342 oasc.Overseer.start Overseer (id=93613545977020419-127.0.0.1:23759_-n_0000000000) starting
   [junit4]   2> 678929 T2342 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 678933 T2342 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 678933 T2342 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 678934 T2369 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 678933 T2370 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 678936 T2342 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores
   [junit4]   2> 678938 T2342 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, schema=schema.xml, collection=control_collection, transient=false, dataDir=data/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores/collection1, shard=, coreNodeName=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores/collection1/, name=collection1, loadOnStartup=true}
   [junit4]   2> 678938 T2342 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores/collection1/
   [junit4]   2> 678939 T2342 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 678939 T2372 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 678939 T2372 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 678940 T2372 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 678940 T2368 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 678941 T2369 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:23759_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23759",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 0
   [junit4]   2> 678941 T2369 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:23759_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23759",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection"}
   [junit4]   2> 678941 T2369 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 678942 T2369 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 678942 T2368 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 679940 T2372 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 679941 T2372 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 679941 T2372 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 679942 T2372 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 679942 T2372 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 679942 T2372 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores/collection1/'
   [junit4]   2> 679973 T2372 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 679976 T2372 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 679988 T2372 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 680014 T2372 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 680070 T2372 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 680172 T2372 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 680183 T2372 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 680625 T2372 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 680642 T2372 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 680646 T2372 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 680655 T2372 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 680660 T2372 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 680664 T2372 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 680665 T2372 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 680666 T2372 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 680666 T2372 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 680667 T2372 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 680667 T2372 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 680667 T2372 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 680667 T2372 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 680668 T2372 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores/collection1/data/
   [junit4]   2> 680668 T2372 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@542890fb
   [junit4]   2> 680669 T2372 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores/collection1/data
   [junit4]   2> 680669 T2372 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores/collection1/data/index/
   [junit4]   2> 680669 T2372 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 680670 T2372 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/control-001/cores/collection1/data/index
   [junit4]   2> 680670 T2372 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=13, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 680671 T2372 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@a0b4a2f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6b3358cf),segFN=segments_1,generation=1}
   [junit4]   2> 680671 T2372 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 680678 T2372 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 680678 T2372 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 680679 T2372 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 680679 T2372 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 680679 T2372 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 680679 T2372 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 680680 T2372 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 680680 T2372 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 680680 T2372 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 680700 T2372 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 680703 T2372 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 680705 T2372 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 680708 T2372 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 680713 T2372 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /replication,/admin/system,/update/json,/update,/update/json/docs,/admin/segments,/admin/luke,/admin/file,/schema,/get,/admin/mbeans,standard,/admin/threads,/admin/plugins,/admin/properties,/update/csv,/admin/ping,/config,/admin/logging
   [junit4]   2> 680715 T2372 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 680717 T2372 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 680717 T2372 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 680718 T2372 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 680719 T2372 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 680720 T2372 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1609071471, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 680721 T2372 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@a0b4a2f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6b3358cf),segFN=segments_1,generation=1}
   [junit4]   2> 680721 T2372 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 680721 T2372 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@4ab3b662[collection1] main
   [junit4]   2> 680722 T2372 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 680722 T2372 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 680723 T2372 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 680723 T2372 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 680723 T2372 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 680723 T2372 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 680724 T2372 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 680724 T2372 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 680724 T2372 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 680725 T2372 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 680725 T2373 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4ab3b662[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 680726 T2372 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 680727 T2376 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:23759 collection:control_collection shard:shard1
   [junit4]   2> 680727 T2342 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 680727 T2342 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 680727 T2376 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 680729 T2342 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 680730 T2342 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 680731 T2380 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@471e5cb name:ZooKeeperConnection Watcher:127.0.0.1:18161/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 680731 T2342 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 680732 T2376 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 680732 T2342 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 680732 T2342 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 680733 T2368 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 680733 T2376 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 680733 T2376 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C556 name=collection1 org.apache.solr.core.SolrCore@e402c82 url=http://127.0.0.1:23759/collection1 node=127.0.0.1:23759_ C556_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:23759_, state=down, base_url=http://127.0.0.1:23759, core=collection1}
   [junit4]   2> 680734 T2376 C:control_collection S:shard1 C556 P23759 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:23759/collection1/
   [junit4]   2> 680734 T2369 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 680734 T2376 C:control_collection S:shard1 C556 P23759 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 680734 T2376 C:control_collection S:shard1 C556 P23759 oasc.SyncStrategy.syncToMe http://127.0.0.1:23759/collection1/ has no replicas
   [junit4]   2> 680734 T2376 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:23759/collection1/ shard1
   [junit4]   2> 680734 T2342 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 680735 T2376 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 680735 T2342 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 680735 T2380 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 680735 T2342 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 680735 T2368 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 680736 T2342 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 680738 T2383 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@57707db6 name:ZooKeeperConnection Watcher:127.0.0.1:18161/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 680738 T2342 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 680738 T2342 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 680740 T2368 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 680741 T2369 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 2, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23759",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 680742 T2369 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 680743 T2369 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 680743 T2380 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 680743 T2368 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 680746 T2368 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 680746 T2369 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 680747 T2368 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 680747 T2380 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 680790 T2376 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:23759/collection1/ and leader is http://127.0.0.1:23759/collection1/
   [junit4]   2> 680791 T2376 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:23759
   [junit4]   2> 680791 T2376 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 680791 T2376 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 680791 T2376 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 680793 T2368 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 680794 T2369 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:23759_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23759",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 4
   [junit4]   2> 680794 T2369 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:23759_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23759",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection"}
   [junit4]   2> 680898 T2368 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 680898 T2380 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 681807 T2342 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores/collection1
   [junit4]   2> 681810 T2342 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001
   [junit4]   2> 681810 T2342 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 681813 T2342 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:63035
   [junit4]   2> 681813 T2342 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=15820, solrconfig=solrconfig.xml, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/tempDir-001/jetty1, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores}
   [junit4]   2> 681814 T2342 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 681814 T2342 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/'
   [junit4]   2> 681847 T2342 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/solr.xml
   [junit4]   2> 681886 T2342 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores
   [junit4]   2> 681887 T2342 oasc.CoreContainer.<init> New CoreContainer 915242244
   [junit4]   2> 681887 T2342 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/]
   [junit4]   2> 681887 T2342 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/lib
   [junit4]   2> 681888 T2342 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/lib).
   [junit4]   2> 681897 T2342 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 681897 T2342 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 681897 T2342 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 681898 T2342 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 681898 T2342 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 681898 T2342 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 681898 T2342 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 681898 T2342 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 681899 T2342 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 681899 T2342 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 681899 T2342 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 681904 T2342 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 681905 T2342 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 681905 T2342 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 681905 T2342 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 681905 T2342 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:18161/solr
   [junit4]   2> 681906 T2342 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 681906 T2342 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 681907 T2342 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 681908 T2395 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@510500e8 name:ZooKeeperConnection Watcher:127.0.0.1:18161 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 681909 T2342 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 681910 T2342 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 681912 T2342 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 681913 T2398 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2f1aea5 name:ZooKeeperConnection Watcher:127.0.0.1:18161/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 681914 T2342 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 681918 T2342 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 682922 T2342 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:15820_
   [junit4]   2> 682923 T2342 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:15820_
   [junit4]   2> 682926 T2342 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 682929 T2342 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores
   [junit4]   2> 682931 T2342 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {coreNodeName=, loadOnStartup=true, dataDir=data/, name=collection1, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores/collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores/collection1/, shard=, schema=schema.xml, collection=collection1, transient=false}
   [junit4]   2> 682932 T2342 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores/collection1/
   [junit4]   2> 682932 T2342 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 682934 T2399 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 682934 T2399 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 682935 T2368 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 682936 T2399 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 682936 T2399 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 682936 T2369 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:15820_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:15820",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 682936 T2399 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 682937 T2369 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:15820_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:15820",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1"}
   [junit4]   2> 682937 T2369 oasco.ReplicaMutator.updateState Collection already exists with numShards=3
   [junit4]   2> 682937 T2369 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 682937 T2399 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 682937 T2399 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 682938 T2369 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 682939 T2398 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 682939 T2398 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 683938 T2399 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 683939 T2399 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 683939 T2399 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 683939 T2399 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 683940 T2399 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 683940 T2399 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores/collection1/'
   [junit4]   2> 683953 T2399 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 683955 T2399 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 683962 T2399 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 683977 T2399 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 684010 T2399 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 684111 T2399 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 684122 T2399 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 684548 T2399 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 684564 T2399 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 684567 T2399 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 684577 T2399 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 684582 T2399 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 684586 T2399 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 684587 T2399 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 684587 T2399 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 684587 T2399 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 684588 T2399 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 684588 T2399 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 684588 T2399 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 684588 T2399 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 684589 T2399 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 684589 T2399 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@542890fb
   [junit4]   2> 684590 T2399 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores/collection1/data
   [junit4]   2> 684590 T2399 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 684591 T2399 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 684591 T2399 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 684591 T2399 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=13, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 684592 T2399 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2758c73a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@357fc504),segFN=segments_1,generation=1}
   [junit4]   2> 684593 T2399 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 684600 T2399 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 684600 T2399 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 684600 T2399 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 684601 T2399 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 684601 T2399 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 684601 T2399 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 684602 T2399 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 684602 T2399 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 684602 T2399 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 684621 T2399 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 684624 T2399 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 684626 T2399 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 684628 T2399 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 684635 T2399 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /replication,/admin/system,/update/json,/update,/update/json/docs,/admin/segments,/admin/luke,/admin/file,/schema,/get,/admin/mbeans,standard,/admin/threads,/admin/plugins,/admin/properties,/update/csv,/admin/ping,/config,/admin/logging
   [junit4]   2> 684637 T2399 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 684639 T2399 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 684639 T2399 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 684641 T2399 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 684641 T2399 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 684642 T2399 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1609071471, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 684643 T2399 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2758c73a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@357fc504),segFN=segments_1,generation=1}
   [junit4]   2> 684643 T2399 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 684643 T2399 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@ebe3283[collection1] main
   [junit4]   2> 684644 T2399 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 684644 T2399 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 684645 T2399 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 684645 T2399 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 684645 T2399 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 684645 T2399 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 684646 T2399 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 684646 T2399 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 684646 T2399 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 684647 T2399 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 684648 T2400 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@ebe3283[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 684648 T2399 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 684649 T2403 C:collection1 S:shard2 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:15820 collection:collection1 shard:shard2
   [junit4]   2> 684649 T2342 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 684650 T2342 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 684650 T2403 C:collection1 S:shard2 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 684654 T2403 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 684656 T2368 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 684656 T2403 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 684656 T2403 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C557 name=collection1 org.apache.solr.core.SolrCore@6815492e url=http://127.0.0.1:15820/collection1 node=127.0.0.1:15820_ C557_STATE=coll:collection1 core:collection1 props:{node_name=127.0.0.1:15820_, state=down, base_url=http://127.0.0.1:15820, core=collection1}
   [junit4]   2> 684656 T2403 C:collection1 S:shard2 C557 P15820 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:15820/collection1/
   [junit4]   2> 684656 T2369 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 684656 T2403 C:collection1 S:shard2 C557 P15820 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 684657 T2403 C:collection1 S:shard2 C557 P15820 oasc.SyncStrategy.syncToMe http://127.0.0.1:15820/collection1/ has no replicas
   [junit4]   2> 684657 T2403 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:15820/collection1/ shard2
   [junit4]   2> 684657 T2403 C:collection1 S:shard2 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 684657 T2369 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 684658 T2398 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 684659 T2398 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 684661 T2368 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 684661 T2369 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:15820",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 684662 T2369 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 684663 T2398 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 684663 T2398 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 684712 T2403 C:collection1 S:shard2 oasc.ZkController.register We are http://127.0.0.1:15820/collection1/ and leader is http://127.0.0.1:15820/collection1/
   [junit4]   2> 684713 T2403 C:collection1 S:shard2 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:15820
   [junit4]   2> 684713 T2403 C:collection1 S:shard2 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 684714 T2403 C:collection1 S:shard2 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 684714 T2403 C:collection1 S:shard2 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 684715 T2368 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 684716 T2369 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:15820_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:15820",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 684717 T2369 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:15820_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:15820",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1"}
   [junit4]   2> 684719 T2369 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 3
   [junit4]   2> 684719 T2403 C:collection1 S:shard2 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 684720 T2398 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 684720 T2398 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 4 
   [junit4]   2> 686203 T2342 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-2-001/cores/collection1
   [junit4]   2> 686206 T2342 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-2-001
   [junit4]   2> 686206 T2342 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 686209 T2342 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:26237
   [junit4]   2> 686209 T2342 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=26236, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/tempDir-001/jetty2, hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-2-001/cores, solrconfig=solrconfig.xml}
   [junit4]   2> 686210 T2342 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 686210 T2342 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-2-001/'
   [junit4]   2> 686243 T2342 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-2-001/solr.xml
   [junit4]   2> 686284 T2342 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-2-001/cores
   [junit4]   2> 686285 T2342 oasc.CoreContainer.<init> New CoreContainer 1800781633
   [junit4]   2> 686285 T2342 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-2-001/]
   [junit4]   2> 686285 T2342 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-2-001/lib
   [junit4]   2> 686286 T2342 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 2902CDAD2A4D727B-001/shard-2-001/lib).
   [junit4]   2> 686295 T2342 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 686296 T2342 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 686296 T2342 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 686296 T2342 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 686296 T2342 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 686297 T2342 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 686297 T2342 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 686297 T2342 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 686297 T2342 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 686298 T2342 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 686298 T2342 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 686303 T2342 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 686304 T2342 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 686304 T2342 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFa

[...truncated too long message...]

lr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 2902CDAD2A4D727B-001/tempDir-001/shard2/cores/collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 2902CDAD2A4D727B-001/tempDir-001/shard2/cores/collection1/data;done=false>>]
   [junit4]   2> 2026438 T7432 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 2902CDAD2A4D727B-001/tempDir-001/shard2/cores/collection1/data
   [junit4]   2> 2026439 T7432 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=DistributedMLTComponentTest -Dtests.method=test -Dtests.seed=2902CDAD2A4D727B -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ru -Dtests.timezone=Asia/Chungking -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR    104s J0 | DistributedMLTComponentTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:35385//collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([2902CDAD2A4D727B:A156F27784B11F83]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:566)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:943)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:558)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:606)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:588)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:567)
   [junit4]    > 	at org.apache.solr.handler.component.DistributedMLTComponentTest.test(DistributedMLTComponentTest.java:126)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: java.net.SocketTimeoutException: Read timed out
   [junit4]    > 	at java.net.SocketInputStream.socketRead0(Native Method)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:152)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:122)
   [junit4]    > 	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
   [junit4]    > 	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
   [junit4]    > 	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 51 more
   [junit4]   2> 2026642 T7432 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 2902CDAD2A4D727B-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {lowerfilt=PostingsFormat(name=MockRandom), intDefault=Lucene50(blocksize=128), lowerfilt1=PostingsFormat(name=Memory doPackFST= false), multiDefault=PostingsFormat(name=MockRandom), range_facet_l=PostingsFormat(name=MockRandom), timestamp=PostingsFormat(name=MockRandom), id=FSTOrd50, _version_=Lucene50(blocksize=128), lowerfilt1and2=PostingsFormat(name=MockRandom)}, docValues:{timestamp=DocValuesFormat(name=Lucene50)}, sim=RandomSimilarityProvider(queryNorm=true,coord=crazy): {}, locale=ru, timezone=Asia/Chungking
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=229808104,total=371720192
   [junit4]   2> NOTE: All tests run in this JVM: [TestRecoveryHdfs, SpellCheckComponentTest, SynonymTokenizerTest, OverseerRolesTest, SpatialHeatmapFacetsTest, TestChildDocTransformer, TestSchemaVersionResource, XmlUpdateRequestHandlerTest, BJQParserTest, TestBM25SimilarityFactory, TestExtendedDismaxParser, AddSchemaFieldsUpdateProcessorFactoryTest, TestOrdValues, ClusterStateUpdateTest, TestFastOutputStream, DeleteShardTest, TestHdfsUpdateLog, UpdateRequestProcessorFactoryTest, DirectSolrSpellCheckerTest, CoreMergeIndexesAdminHandlerTest, CoreAdminRequestStatusTest, TestLMDirichletSimilarityFactory, HdfsUnloadDistributedZkTest, SystemInfoHandlerTest, TestSimpleTrackingShardHandler, CloudMLTQParserTest, DistanceUnitsTest, TestReqParamsAPI, DisMaxRequestHandlerTest, AtomicUpdatesTest, TestDocBasedVersionConstraints, QueryResultKeyTest, TestUniqueKeyFieldResource, TestAnalyzedSuggestions, SpellCheckCollatorTest, TestStressLucene, TestCodecSupport, SchemaVersionSpecificBehaviorTest, TermsComponentTest, OverriddenZkACLAndCredentialsProvidersTest, DistributedSuggestComponentTest, SolrInfoMBeanTest, TestFieldSortValues, AddBlockUpdateTest, TestReplicaProperties, TestBlobHandler, TestFuzzyAnalyzedSuggestions, OutputWriterTest, BadCopyFieldTest, TestBinaryResponseWriter, DistributedFacetPivotSmallAdvancedTest, TestDefaultSearchFieldResource, CachingDirectoryFactoryTest, FileBasedSpellCheckerTest, TestJsonFacets, DOMUtilTest, RollingRestartTest, HttpPartitionTest, ShardSplitTest, TestCloudSchemaless, FullSolrCloudDistribCmdsTest, TestSolrConfigHandlerConcurrent, TestDynamicLoading, CollectionReloadTest, TestRebalanceLeaders, CloudExitableDirectoryReaderTest, OverseerStatusTest, TestDistribDocBasedVersion, TestShortCircuitedRequests, DistributedFacetPivotLargeTest, ZkSolrClientTest, TestHashPartitioner, TestReload, ZkCLITest, DistributedMLTComponentTest]
   [junit4] Completed on J0 in 103.81s, 1 test, 1 error <<< FAILURES!

[...truncated 413 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 479 suites, 1903 tests, 2 errors, 61 ignored (21 assumptions)

Total time: 58 minutes 57 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 212829115 bytes
Compression is 0.0%
Took 39 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2905 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2905/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:21000/p_se/a/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:21000/p_se/a/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([DFC12658F5827783:579519825B7E1A7B]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:21000 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28440/_xv/b/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28440/_xv/b/collection1
	at __randomizedtesting.SeedInfo.seed([DFC12658F5827783:579519825B7E1A7B]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28440/_xv/b/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:28440 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22580/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22580/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([DFC12658F5827783:579519825B7E1A7B]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22580/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:22580 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9544 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/init-core-data-001
   [junit4]   2> 228752 T610 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 228762 T610 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 228763 T611 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 228862 T610 oasc.ZkTestServer.run start zk server on port:22563
   [junit4]   2> 228863 T610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 228864 T610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 228873 T618 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@37715099 name:ZooKeeperConnection Watcher:127.0.0.1:22563 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 228873 T610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 228874 T610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 228874 T610 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 228878 T610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 228879 T610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 228882 T621 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@42f758cf name:ZooKeeperConnection Watcher:127.0.0.1:22563/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 228882 T610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 228883 T610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 228883 T610 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 228886 T610 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 228888 T610 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 228890 T610 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 228893 T610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 228895 T610 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 228900 T610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 228901 T610 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 228905 T610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 228906 T610 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 228909 T610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 228911 T610 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 228915 T610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 228916 T610 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 228920 T610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 228922 T610 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 228927 T610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 228929 T610 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 228933 T610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 228935 T610 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 228939 T610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 228940 T610 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 228947 T610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 228949 T610 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 228954 T610 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 228955 T610 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 233138 T610 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores/collection1
   [junit4]   2> 233142 T610 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 233145 T610 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:22581
   [junit4]   2> 233145 T610 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/tempDir-001/control/data, hostContext=/, hostPort=22580, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores}
   [junit4]   2> 233146 T610 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 233146 T610 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/'
   [junit4]   2> 233194 T610 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/solr.xml
   [junit4]   2> 233407 T610 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores
   [junit4]   2> 233408 T610 oasc.CoreContainer.<init> New CoreContainer 957384846
   [junit4]   2> 233408 T610 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/]
   [junit4]   2> 233409 T610 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/lib
   [junit4]   2> 233413 T610 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/lib).
   [junit4]   2> 233446 T610 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 233446 T610 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 233447 T610 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 233447 T610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 233447 T610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 233448 T610 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 233448 T610 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 233449 T610 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 233450 T610 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 233450 T610 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 233450 T610 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 233475 T610 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 233478 T610 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 233478 T610 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 233478 T610 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 233479 T610 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:22563/solr
   [junit4]   2> 233479 T610 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 233480 T610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 233482 T610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 233484 T633 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2258256e name:ZooKeeperConnection Watcher:127.0.0.1:22563 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 233485 T610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 233486 T610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 233494 T610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 233496 T636 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6b2e0f34 name:ZooKeeperConnection Watcher:127.0.0.1:22563/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 233496 T610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 233499 T610 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 233506 T610 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 233512 T610 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 233517 T610 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 233521 T610 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 233534 T610 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 233542 T610 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 233547 T610 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 233552 T610 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:22580_
   [junit4]   2> 233553 T610 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:22580_
   [junit4]   2> 233557 T610 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 233560 T610 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 233565 T610 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 233569 T610 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:22580_
   [junit4]   2> 233569 T610 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 233574 T610 oasc.Overseer.start Overseer (id=93612242919948291-127.0.0.1:22580_-n_0000000000) starting
   [junit4]   2> 233579 T610 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 233594 T610 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 233595 T638 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 233595 T610 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 233596 T637 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 233603 T610 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores
   [junit4]   2> 233611 T610 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {loadOnStartup=true, coreNodeName=, schema=schema.xml, transient=false, dataDir=data/, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores/collection1/, shard=, config=solrconfig.xml, collection=control_collection, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores/collection1, name=collection1}
   [junit4]   2> 233612 T610 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores/collection1/
   [junit4]   2> 233612 T610 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 233614 T640 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 233614 T640 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 233616 T636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 233616 T640 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 233618 T637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22580",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:22580_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null} current state version: 0
   [junit4]   2> 233620 T637 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22580",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:22580_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 233620 T637 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 233620 T637 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 233624 T636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 234617 T640 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 234617 T640 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 234619 T640 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 234620 T640 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 234621 T640 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 234621 T640 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores/collection1/'
   [junit4]   2> 234651 T640 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 234658 T640 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 234686 T640 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 234753 T640 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 234888 T640 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 234890 T640 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 234913 T640 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 235913 T640 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 235952 T640 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 235957 T640 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 235977 T640 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 235986 T640 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 235993 T640 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 235995 T640 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 235996 T640 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 235997 T640 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 235998 T640 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 235998 T640 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 235999 T640 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 235999 T640 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 236000 T640 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores/collection1/data/
   [junit4]   2> 236000 T640 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7f14be2e
   [junit4]   2> 236001 T640 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores/collection1/data
   [junit4]   2> 236002 T640 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores/collection1/data/index/
   [junit4]   2> 236002 T640 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 236003 T640 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/control-001/cores/collection1/data/index
   [junit4]   2> 236004 T640 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=34, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6812259126929583]
   [junit4]   2> 236005 T640 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7f765e23 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@341e8d35),segFN=segments_1,generation=1}
   [junit4]   2> 236005 T640 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 236014 T640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 236014 T640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 236015 T640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 236015 T640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 236015 T640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 236016 T640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 236017 T640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 236017 T640 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 236017 T640 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 236039 T640 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 236041 T640 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 236043 T640 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 236045 T640 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 236049 T640 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /schema,/admin/system,/admin/file,/admin/logging,/admin/luke,standard,/update/csv,/admin/ping,/update/json,/admin/threads,/admin/mbeans,/admin/properties,/config,/get,/admin/segments,/update,/replication,/update/json/docs,/admin/plugins
   [junit4]   2> 236051 T640 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 236053 T640 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 236053 T640 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 236055 T640 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 236055 T640 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 236055 T640 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=46, maxMergeAtOnceExplicit=32, maxMergedSegmentMB=50.4541015625, floorSegmentMB=0.912109375, forceMergeDeletesPctAllowed=9.525263553538434, segmentsPerTier=24.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.16973219653750882
   [junit4]   2> 236056 T640 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7f765e23 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@341e8d35),segFN=segments_1,generation=1}
   [junit4]   2> 236056 T640 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 236057 T640 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@6b9df4b3[collection1] main
   [junit4]   2> 236057 T640 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 236058 T640 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 236058 T640 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 236059 T640 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 236059 T640 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 236059 T640 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 236060 T640 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 236060 T640 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 236060 T640 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 236061 T640 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 236061 T641 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6b9df4b3[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 236062 T640 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 236063 T644 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:22580 collection:control_collection shard:shard1
   [junit4]   2> 236063 T610 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 236063 T610 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 236063 T644 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 236067 T644 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 236068 T610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 236068 T636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 236068 T644 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 236068 T610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 236069 T644 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C67 name=collection1 org.apache.solr.core.SolrCore@347de428 url=http://127.0.0.1:22580/collection1 node=127.0.0.1:22580_ C67_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:22580, core=collection1, node_name=127.0.0.1:22580_}
   [junit4]   2> 236069 T644 C:control_collection S:shard1 C67 P22580 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:22580/collection1/
   [junit4]   2> 236070 T644 C:control_collection S:shard1 C67 P22580 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 236070 T644 C:control_collection S:shard1 C67 P22580 oasc.SyncStrategy.syncToMe http://127.0.0.1:22580/collection1/ has no replicas
   [junit4]   2> 236070 T637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 236070 T648 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4b4907d8 name:ZooKeeperConnection Watcher:127.0.0.1:22563/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 236070 T644 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:22580/collection1/ shard1
   [junit4]   2> 236070 T610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 236071 T644 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 236071 T636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 236071 T610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 236072 T610 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 236074 T636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 236074 T610 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 236075 T637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22580",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 236178 T636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 236178 T648 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 236227 T644 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:22580/collection1/ and leader is http://127.0.0.1:22580/collection1/
   [junit4]   2> 236228 T644 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:22580
   [junit4]   2> 236228 T644 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 236228 T644 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 236228 T644 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 236230 T636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 236231 T637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22580",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:22580_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null} current state version: 3
   [junit4]   2> 236231 T637 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22580",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:22580_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 236334 T636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 236334 T648 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 239237 T610 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores/collection1
   [junit4]   2> 239249 T610 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001
   [junit4]   2> 239251 T610 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 239259 T610 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:22602
   [junit4]   2> 239259 T610 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/tempDir-001/jetty1, hostContext=/, solrconfig=solrconfig.xml, hostPort=22601}
   [junit4]   2> 239260 T610 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 239261 T610 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/'
   [junit4]   2> 239405 T610 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/solr.xml
   [junit4]   2> 239472 T610 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores
   [junit4]   2> 239472 T610 oasc.CoreContainer.<init> New CoreContainer 1684710647
   [junit4]   2> 239473 T610 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/]
   [junit4]   2> 239473 T610 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/lib
   [junit4]   2> 239474 T610 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/lib).
   [junit4]   2> 239487 T610 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 239488 T610 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 239488 T610 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 239489 T610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 239489 T610 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 239489 T610 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 239489 T610 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 239490 T610 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 239490 T610 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 239490 T610 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 239491 T610 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 239497 T610 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 239498 T610 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 239498 T610 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 239499 T610 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 239499 T610 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:22563/solr
   [junit4]   2> 239500 T610 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 239500 T610 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 239501 T610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 239504 T660 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@74ed31e5 name:ZooKeeperConnection Watcher:127.0.0.1:22563 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 239504 T610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 239505 T610 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 239509 T610 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 239512 T663 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4faae9f3 name:ZooKeeperConnection Watcher:127.0.0.1:22563/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 239512 T610 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 239518 T610 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 240529 T610 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:22601_
   [junit4]   2> 240531 T610 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:22601_
   [junit4]   2> 240543 T610 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 240551 T610 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores
   [junit4]   2> 240560 T610 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {transient=false, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores/collection1/, collection=collection1, name=collection1, shard=, loadOnStartup=true, schema=schema.xml, coreNodeName=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores/collection1, config=solrconfig.xml, dataDir=data/}
   [junit4]   2> 240561 T610 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores/collection1/
   [junit4]   2> 240562 T610 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 240563 T664 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 240564 T664 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 240566 T664 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 240566 T636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 240567 T637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22601",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:22601_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null} current state version: 4
   [junit4]   2> 240568 T637 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22601",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:22601_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 240568 T637 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 240569 T637 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard3
   [junit4]   2> 240572 T636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 240572 T648 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 240573 T663 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 241566 T664 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 241567 T664 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 241568 T664 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 241569 T664 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 241569 T664 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 241570 T664 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores/collection1/'
   [junit4]   2> 241604 T664 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 241615 T664 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 241641 T664 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 241698 T664 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 241785 T664 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 241787 T664 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 241807 T664 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 242758 T664 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 242775 T664 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 242778 T664 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 242812 T664 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 242817 T664 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 242821 T664 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 242822 T664 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 242822 T664 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 242823 T664 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 242823 T664 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 242823 T664 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 242824 T664 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 242824 T664 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 242824 T664 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 242825 T664 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7f14be2e
   [junit4]   2> 242825 T664 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores/collection1/data
   [junit4]   2> 242826 T664 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 242826 T664 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 242826 T664 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 242827 T664 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=34, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6812259126929583]
   [junit4]   2> 242828 T664 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3d1ff8dc lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7e0b52dd),segFN=segments_1,generation=1}
   [junit4]   2> 242828 T664 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 242834 T664 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 242834 T664 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 242835 T664 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 242835 T664 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 242835 T664 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 242835 T664 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 242836 T664 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 242836 T664 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 242836 T664 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 242853 T664 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 242855 T664 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 242857 T664 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 242859 T664 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 242867 T664 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /schema,/admin/system,/admin/file,/admin/logging,/admin/luke,standard,/update/csv,/admin/ping,/update/json,/admin/threads,/admin/mbeans,/admin/properties,/config,/get,/admin/segments,/update,/replication,/update/json/docs,/admin/plugins
   [junit4]   2> 242869 T664 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 242871 T664 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 242871 T664 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 242873 T664 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 242873 T664 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 242874 T664 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=46, maxMergeAtOnceExplicit=32, maxMergedSegmentMB=50.4541015625, floorSegmentMB=0.912109375, forceMergeDeletesPctAllowed=9.525263553538434, segmentsPerTier=24.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.16973219653750882
   [junit4]   2> 242876 T664 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3d1ff8dc lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7e0b52dd),segFN=segments_1,generation=1}
   [junit4]   2> 242876 T664 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 242876 T664 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@63471445[collection1] main
   [junit4]   2> 242877 T664 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 242880 T664 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 242880 T664 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 242884 T664 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 242884 T664 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 242884 T664 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 242885 T664 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 242885 T664 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 242886 T664 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 242886 T664 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 242888 T665 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@63471445[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 242889 T664 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 242890 T668 C:collection1 S:shard3 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:22601 collection:collection1 shard:shard3
   [junit4]   2> 242891 T610 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 242892 T610 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 242892 T668 C:collection1 S:shard3 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard3/election
   [junit4]   2> 242905 T668 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard3
   [junit4]   2> 242908 T636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 242909 T668 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 242909 T668 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C68 name=collection1 org.apache.solr.core.SolrCore@65a55801 url=http://127.0.0.1:22601/collection1 node=127.0.0.1:22601_ C68_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:22601, core=collection1, node_name=127.0.0.1:22601_}
   [junit4]   2> 242910 T668 C:collection1 S:shard3 C68 P22601 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:22601/collection1/
   [junit4]   2> 242910 T668 C:collection1 S:shard3 C68 P22601 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 242910 T637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 242911 T668 C:collection1 S:shard3 C68 P22601 oasc.SyncStrategy.syncToMe http://127.0.0.1:22601/collection1/ has no replicas
   [junit4]   2> 242912 T668 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:22601/collection1/ shard3
   [junit4]   2> 242912 T668 C:collection1 S:shard3 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard3
   [junit4]   2> 242918 T648 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 242919 T663 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 242918 T636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 242924 T636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 242926 T637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22601",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 243035 T636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 243035 T648 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 243035 T663 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 243079 T668 C:collection1 S:shard3 oasc.ZkController.register We are http://127.0.0.1:22601/collection1/ and leader is http://127.0.0.1:22601/collection1/
   [junit4]   2> 243079 T668 C:collection1 S:shard3 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:22601
   [junit4]   2> 243080 T668 C:collection1 S:shard3 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 243080 T668 C:collection1 S:shard3 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 243080 T668 C:collection1 S:shard3 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 243083 T636 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 243085 T637 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22601",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:22601_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null} current state version: 7
   [junit4]   2> 243086 T637 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22601",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:22601_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 243194 T636 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 243195 T648 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 243194 T663 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 247015 T610 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest DFC12658F5827783-001/shard-2-001/cores/collection1
   [junit4]   2> 247025 T610 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/sol

[...truncated too long message...]

etContextHandler{/p_se/a,null}
   [junit4]   2> 1487445 T2508 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:20927 20927
   [junit4]   2> 1487791 T2509 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:20927 20927
   [junit4]   2> 1487793 T2509 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/aliases.json
   [junit4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 		3	/solr/collections/c8n_1x3_commits/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 		4	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1487793 T2508 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:21008/p_se/a
   [junit4]   2> 1487793 T2508 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 11 connections to: http://127.0.0.1:20963/p_se/a
   [junit4]   2> 1487794 T2508 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:20980/p_se/a
   [junit4]   2> 1487794 T2508 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:21000/p_se/a
   [junit4]   2> 1487794 T2508 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:20951/p_se/a
   [junit4]   2> 1487795 T2508 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:21008/p_se/a
   [junit4]   2> 1487795 T2508 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:20963/p_se/a
   [junit4]   2> 1487795 T2508 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:20980/p_se/a
   [junit4]   2> 1487795 T2508 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:21000/p_se/a
   [junit4]   2> 1487796 T2508 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:20951/p_se/a
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderInitiatedRecoveryOnCommitTest -Dtests.method=test -Dtests.seed=DFC12658F5827783 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=en_PH -Dtests.timezone=America/Paramaribo -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   92.8s J0 | LeaderInitiatedRecoveryOnCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:21000/p_se/a/c8n_1x3_commits_shard1_replica1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([DFC12658F5827783:579519825B7E1A7B]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:21000 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 48 more
   [junit4]   2> 1487804 T2508 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DFC12658F5827783-001
   [junit4]   2> 92813 T2507 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=DefaultSimilarity, locale=en_PH, timezone=America/Paramaribo
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=85241336,total=286261248
   [junit4]   2> NOTE: All tests run in this JVM: [SolrInfoMBeanTest, JsonLoaderTest, TestJmxIntegration, ShardRoutingTest, ExitableDirectoryReaderTest, CSVRequestHandlerTest, TestSweetSpotSimilarityFactory, TestSuggestSpellingConverter, HighlighterMaxOffsetTest, OverseerRolesTest, TestLuceneMatchVersion, ReplicationFactorTest, TestShortCircuitedRequests, DateRangeFieldTest, DocumentBuilderTest, CacheHeaderTest, RecoveryZkTest, TestInitParams, IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestJettySolrRunner, TestSerializedLuceneMatchVersion, TestSchemaResource, TestTrackingShardHandlerFactory, PathHierarchyTokenizerFactoryTest, TestDynamicFieldResource, BasicFunctionalityTest, RequiredFieldsTest, SimpleCollectionCreateDeleteTest, TestExactSharedStatsCache, TestFileDictionaryLookup, TestFunctionQuery, TestClusterStateMutator, TestManagedResourceStorage, AnalysisErrorHandlingTest, RemoteQueryErrorTest, TestSolrDeletionPolicy2, HttpPartitionTest, AnalyticsQueryTest, TestTrie, XsltUpdateRequestHandlerTest, TestConfigOverlay, TestInitQParser, ScriptEngineTest, TestCollapseQParserPlugin, DirectUpdateHandlerTest, TestQueryTypes, TestRecovery, SolrIndexSplitterTest, AliasIntegrationTest, TestReloadAndDeleteDocs, HdfsDirectoryTest, LeaderInitiatedRecoveryOnCommitTest]
   [junit4] Completed on J0 in 93.60s, 1 test, 1 error <<< FAILURES!

[...truncated 748 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 479 suites, 1903 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 61 minutes 4 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 213820630 bytes
Compression is 0.0%
Took 41 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2904 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2904/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:32152/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:32152/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([998831C6A7E2310A:11DC0E1C091E5CF2]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:32152 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37941/cpwc/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37941/cpwc/collection1
	at __randomizedtesting.SeedInfo.seed([998831C6A7E2310A:11DC0E1C091E5CF2]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37941/cpwc/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:37941 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:31600/_bkj/mq/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:31600/_bkj/mq/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([998831C6A7E2310A:11DC0E1C091E5CF2]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:31600/_bkj/mq/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:31600 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9355 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/init-core-data-001
   [junit4]   2> 160347 T395 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 160352 T395 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 160353 T396 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 160453 T395 oasc.ZkTestServer.run start zk server on port:32104
   [junit4]   2> 160454 T395 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 160455 T395 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 160458 T403 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6c3d4223 name:ZooKeeperConnection Watcher:127.0.0.1:32104 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 160458 T395 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 160459 T395 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 160459 T395 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 160462 T395 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 160462 T395 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 160463 T406 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6c4b078f name:ZooKeeperConnection Watcher:127.0.0.1:32104/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 160464 T395 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 160464 T395 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 160464 T395 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 160466 T395 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 160468 T395 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 160469 T395 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 160471 T395 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 160472 T395 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 160475 T395 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 160475 T395 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 160478 T395 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 160478 T395 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 160480 T395 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 160480 T395 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 160483 T395 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 160483 T395 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 160485 T395 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 160486 T395 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 160488 T395 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 160489 T395 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 160490 T395 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 160491 T395 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 160492 T395 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 160493 T395 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 160495 T395 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 160495 T395 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 160497 T395 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 160497 T395 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 161891 T395 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores/collection1
   [junit4]   2> 161896 T395 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 161900 T395 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:32111
   [junit4]   2> 161900 T395 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=32110, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/tempDir-001/control/data}
   [junit4]   2> 161901 T395 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 161901 T395 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/'
   [junit4]   2> 161953 T395 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/solr.xml
   [junit4]   2> 162016 T395 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores
   [junit4]   2> 162017 T395 oasc.CoreContainer.<init> New CoreContainer 139305003
   [junit4]   2> 162017 T395 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/]
   [junit4]   2> 162018 T395 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/lib
   [junit4]   2> 162019 T395 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/lib).
   [junit4]   2> 162033 T395 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 162033 T395 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 162033 T395 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 162034 T395 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 162034 T395 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 162034 T395 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 162035 T395 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 162035 T395 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 162035 T395 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 162035 T395 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 162036 T395 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 162043 T395 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 162045 T395 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 162045 T395 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 162045 T395 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 162046 T395 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:32104/solr
   [junit4]   2> 162046 T395 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 162046 T395 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 162047 T395 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 162049 T418 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7a84a043 name:ZooKeeperConnection Watcher:127.0.0.1:32104 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 162049 T395 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 162050 T395 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 162052 T395 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 162054 T421 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@22497601 name:ZooKeeperConnection Watcher:127.0.0.1:32104/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 162054 T395 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 162056 T395 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 162059 T395 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 162061 T395 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 162063 T395 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 162066 T395 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 162068 T395 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 162070 T395 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 162072 T395 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 162073 T395 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:32110_
   [junit4]   2> 162074 T395 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:32110_
   [junit4]   2> 162076 T395 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 162077 T395 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 162079 T395 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 162081 T395 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:32110_
   [junit4]   2> 162081 T395 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 162083 T395 oasc.Overseer.start Overseer (id=93611275814240259-127.0.0.1:32110_-n_0000000000) starting
   [junit4]   2> 162085 T395 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 162092 T395 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 162093 T423 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 162093 T395 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 162093 T422 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 162098 T395 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores
   [junit4]   2> 162101 T395 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores/collection1/, shard=, name=collection1, schema=schema.xml, config=solrconfig.xml, loadOnStartup=true, transient=false, collection=control_collection, coreNodeName=, dataDir=data/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores/collection1}
   [junit4]   2> 162101 T395 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores/collection1/
   [junit4]   2> 162101 T395 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 162103 T425 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 162103 T425 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 162104 T421 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 162104 T425 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 162105 T422 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32110",
   [junit4]   2> 	  "node_name":"127.0.0.1:32110_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null} current state version: 0
   [junit4]   2> 162106 T422 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32110",
   [junit4]   2> 	  "node_name":"127.0.0.1:32110_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 162106 T422 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 162106 T422 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 162108 T421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 163105 T425 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 163106 T425 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 163107 T425 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 163107 T425 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 163108 T425 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 163108 T425 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores/collection1/'
   [junit4]   2> 163127 T425 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 163130 T425 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 163145 T425 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 163175 T425 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 163278 T425 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 163280 T425 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 163297 T425 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 163690 T425 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 163703 T425 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 163706 T425 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 163722 T425 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 163727 T425 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 163730 T425 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 163731 T425 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 163732 T425 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 163732 T425 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 163733 T425 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 163733 T425 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 163733 T425 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 163733 T425 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 163734 T425 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores/collection1/data/
   [junit4]   2> 163734 T425 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4f3fd958
   [junit4]   2> 163734 T425 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores/collection1/data
   [junit4]   2> 163735 T425 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores/collection1/data/index/
   [junit4]   2> 163735 T425 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 163735 T425 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/control-001/cores/collection1/data/index
   [junit4]   2> 163736 T425 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=73840210, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 163736 T425 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@270b6d1c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5f285d4a),segFN=segments_1,generation=1}
   [junit4]   2> 163736 T425 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 163741 T425 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 163741 T425 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 163741 T425 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 163742 T425 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 163742 T425 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 163742 T425 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 163742 T425 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 163742 T425 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 163743 T425 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 163757 T425 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 163759 T425 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 163760 T425 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 163762 T425 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 163766 T425 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/luke,/replication,/get,/admin/mbeans,/admin/properties,/update/csv,/update/json,/admin/logging,/admin/system,/admin/threads,/update,/admin/ping,/schema,/admin/plugins,/update/json/docs,/admin/segments,/config,standard,/admin/file
   [junit4]   2> 163768 T425 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 163769 T425 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 163769 T425 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 163770 T425 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 163771 T425 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 163771 T425 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=38, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.10602606676023907]
   [junit4]   2> 163771 T425 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@270b6d1c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5f285d4a),segFN=segments_1,generation=1}
   [junit4]   2> 163772 T425 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 163772 T425 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@7a00e70c[collection1] main
   [junit4]   2> 163772 T425 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 163773 T425 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 163773 T425 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 163774 T425 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 163774 T425 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 163774 T425 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 163775 T425 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 163775 T425 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 163775 T425 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 163775 T425 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 163776 T426 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7a00e70c[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 163777 T425 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 163778 T429 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:32110 collection:control_collection shard:shard1
   [junit4]   2> 163778 T395 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 163779 T395 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 163779 T429 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 163784 T429 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 163785 T395 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 163785 T395 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 163786 T421 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 163787 T429 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 163787 T433 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@425a03e7 name:ZooKeeperConnection Watcher:127.0.0.1:32104/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 163787 T429 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 163787 T395 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 163787 T422 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C107 name=collection1 org.apache.solr.core.SolrCore@7125ade5 url=http://127.0.0.1:32110/collection1 node=127.0.0.1:32110_ C107_STATE=coll:control_collection core:collection1 props:{core=collection1, base_url=http://127.0.0.1:32110, node_name=127.0.0.1:32110_, state=down}
   [junit4]   2> 163787 T429 C:control_collection S:shard1 C107 P32110 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:32110/collection1/
   [junit4]   2> 163788 T395 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 163788 T429 C:control_collection S:shard1 C107 P32110 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 163789 T395 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 163789 T429 C:control_collection S:shard1 C107 P32110 oasc.SyncStrategy.syncToMe http://127.0.0.1:32110/collection1/ has no replicas
   [junit4]   2> 163789 T429 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:32110/collection1/ shard1
   [junit4]   2> 163789 T421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 163790 T429 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 163792 T395 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 163795 T421 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 163796 T422 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32110",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 163903 T433 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 163903 T421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 163947 T429 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:32110/collection1/ and leader is http://127.0.0.1:32110/collection1/
   [junit4]   2> 163948 T429 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:32110
   [junit4]   2> 163948 T429 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 163948 T429 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 163948 T429 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 163950 T421 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 163951 T422 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32110",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:32110_",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1"} current state version: 3
   [junit4]   2> 163952 T422 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32110",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:32110_",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1"}
   [junit4]   2> 164055 T421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 164055 T433 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 166294 T395 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores/collection1
   [junit4]   2> 166298 T395 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001
   [junit4]   2> 166299 T395 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 166302 T395 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:32132
   [junit4]   2> 166303 T395 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores, hostPort=32131, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/tempDir-001/jetty1, solrconfig=solrconfig.xml}
   [junit4]   2> 166304 T395 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 166305 T395 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/'
   [junit4]   2> 166365 T395 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/solr.xml
   [junit4]   2> 166431 T395 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores
   [junit4]   2> 166431 T395 oasc.CoreContainer.<init> New CoreContainer 1214137018
   [junit4]   2> 166431 T395 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/]
   [junit4]   2> 166432 T395 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/lib
   [junit4]   2> 166433 T395 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/lib).
   [junit4]   2> 166448 T395 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 166449 T395 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 166449 T395 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 166449 T395 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 166450 T395 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 166450 T395 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 166450 T395 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 166451 T395 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 166451 T395 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 166451 T395 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 166452 T395 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 166460 T395 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 166462 T395 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 166462 T395 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 166462 T395 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 166463 T395 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:32104/solr
   [junit4]   2> 166463 T395 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 166463 T395 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 166464 T395 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 166467 T445 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@48885506 name:ZooKeeperConnection Watcher:127.0.0.1:32104 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 166467 T395 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 166468 T395 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 166471 T395 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 166473 T448 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@b27c214 name:ZooKeeperConnection Watcher:127.0.0.1:32104/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 166474 T395 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 166482 T395 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 167488 T395 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:32131_
   [junit4]   2> 167489 T395 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:32131_
   [junit4]   2> 167491 T395 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 167494 T395 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores
   [junit4]   2> 167496 T395 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, name=collection1, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores/collection1, schema=schema.xml, loadOnStartup=true, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores/collection1/, collection=collection1, dataDir=data/, coreNodeName=, shard=}
   [junit4]   2> 167497 T395 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores/collection1/
   [junit4]   2> 167497 T395 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 167498 T449 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 167498 T449 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 167499 T421 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 167499 T449 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 167500 T422 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32131",
   [junit4]   2> 	  "node_name":"127.0.0.1:32131_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null} current state version: 4
   [junit4]   2> 167500 T422 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32131",
   [junit4]   2> 	  "node_name":"127.0.0.1:32131_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 167500 T422 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 167501 T422 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 167502 T421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 167502 T433 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 167502 T448 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 168499 T449 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 168500 T449 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 168501 T449 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 168501 T449 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 168502 T449 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 168502 T449 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores/collection1/'
   [junit4]   2> 168521 T449 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 168524 T449 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 168538 T449 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 168567 T449 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 168624 T449 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 168626 T449 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 168640 T449 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 169025 T449 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 169039 T449 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 169041 T449 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 169054 T449 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 169058 T449 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 169061 T449 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 169062 T449 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 169062 T449 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 169062 T449 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 169063 T449 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 169063 T449 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 169063 T449 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 169064 T449 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 169064 T449 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 169064 T449 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4f3fd958
   [junit4]   2> 169065 T449 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores/collection1/data
   [junit4]   2> 169065 T449 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 169065 T449 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 169065 T449 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 998831C6A7E2310A-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 169066 T449 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=73840210, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 169067 T449 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1abcd676 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@63a3eab0),segFN=segments_1,generation=1}
   [junit4]   2> 169067 T449 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 169073 T449 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 169073 T449 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 169073 T449 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 169074 T449 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 169074 T449 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 169074 T449 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 169074 T449 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 169075 T449 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 169075 T449 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 169094 T449 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 169096 T449 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 169097 T449 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 169099 T449 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 169105 T449 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/luke,/replication,/get,/admin/mbeans,/admin/properties,/update/csv,/update/json,/admin/logging,/admin/system,/admin/threads,/update,/admin/ping,/schema,/admin/plugins,/update/json/docs,/admin/segments,/config,standard,/admin/file
   [junit4]   2> 169107 T449 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 169108 T449 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 169108 T449 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 169110 T449 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 169110 T449 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 169111 T449 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=38, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.10602606676023907]
   [junit4]   2> 169111 T449 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1abcd676 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@63a3eab0),segFN=segments_1,generation=1}
   [junit4]   2> 169111 T449 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 169111 T449 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@6a3c49d3[collection1] main
   [junit4]   2> 169112 T449 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 169113 T449 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 169113 T449 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 169113 T449 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 169113 T449 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 169114 T449 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 169114 T449 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 169114 T449 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 169114 T449 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 169115 T449 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 169116 T450 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6a3c49d3[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 169116 T449 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 169117 T453 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:32131 collection:collection1 shard:shard1
   [junit4]   2> 169117 T395 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 169118 T395 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 169118 T453 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 169122 T453 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 169123 T421 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 169124 T453 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 169124 T453 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C108 name=collection1 org.apache.solr.core.SolrCore@210cd041 url=http://127.0.0.1:32131/collection1 node=127.0.0.1:32131_ C108_STATE=coll:collection1 core:collection1 props:{core=collection1, base_url=http://127.0.0.1:32131, node_name=127.0.0.1:32131_, state=down}
   [junit4]   2> 169124 T453 C:collection1 S:shard1 C108 P32131 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:32131/collection1/
   [junit4]   2> 169124 T422 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 169125 T453 C:collection1 S:shard1 C108 P32131 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 169125 T453 C:collection1 S:shard1 C108 P32131 oasc.SyncStrategy.syncToMe http://127.0.0.1:32131/collection1/ has no replicas
   [junit4]   2> 169125 T453 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:32131/collection1/ shard1
   [junit4]   2> 169125 T453 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 169126 T433 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 169126 T421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 169126 T448 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 169130 T421 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 169131 T422 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32131",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 169234 T421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 169234 T448 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 169234 T433 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 169283 T453 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:32131/collection1/ and leader is http://127.0.0.1:32131/collection1/
   [junit4]   2> 169284 T453 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:32131
   [junit4]   2> 169284 T453 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 169284 T453 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 169284 T453 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 169285 T421 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 169286 T422 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32131",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:32131_",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1"} current state version: 7
   [junit4]   2> 169287 T422 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32131",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:32131_",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1"}
   [junit4]   2> 169390 T421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 169390 T448 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 169390 T433 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstat

[...truncated too long message...]

ue
   [junit4]   2> 		4	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1504270 T3781 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 10 connections to: http://127.0.0.1:31600/_bkj/mq
   [junit4]   2> 1504271 T3781 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:31574/_bkj/mq
   [junit4]   2> 1504271 T3781 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 1 connections to: http://127.0.0.1:31559/_bkj/mq
   [junit4]   2> 1504271 T3781 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:31585/_bkj/mq
   [junit4]   2> 1504271 T3781 C:control_collection S:shard1 c:collection1 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 1504272 T3781 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:31600/_bkj/mq
   [junit4]   2> 1504272 T3781 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:31574/_bkj/mq
   [junit4]   2> 1504272 T3781 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:31559/_bkj/mq
   [junit4]   2> 1504272 T3781 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:31585/_bkj/mq
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=998831C6A7E2310A -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ro -Dtests.timezone=SystemV/EST5 -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   80.1s J1 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:31600/_bkj/mq/repfacttest_c8n_1x3_shard1_replica1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([998831C6A7E2310A:11DC0E1C091E5CF2]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:31600/_bkj/mq/repfacttest_c8n_1x3_shard1_replica1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 48 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:31600 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 53 more
   [junit4]   2> 1504288 T3781 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 998831C6A7E2310A-001
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarityProvider(queryNorm=false,coord=yes): {}, locale=ro, timezone=SystemV/EST5
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=46987136,total=306708480
   [junit4]   2> NOTE: All tests run in this JVM: [DeleteReplicaTest, TestDynamicFieldResource, SearchHandlerTest, SpellPossibilityIteratorTest, TestSolrCoreProperties, ShardRoutingTest, ExternalFileFieldSortTest, TestAnalyzeInfixSuggestions, RequiredFieldsTest, TestRTGBase, LeaderFailoverAfterPartitionTest, XmlUpdateRequestHandlerTest, LeaderElectionIntegrationTest, DeleteInactiveReplicaTest, TestEmbeddedSolrServerConstructors, AddBlockUpdateTest, TestHashQParserPlugin, TestManagedSchema, TestManagedStopFilterFactory, DocumentAnalysisRequestHandlerTest, TestCSVLoader, TestCursorMarkWithoutUniqueKey, FastVectorHighlighterTest, TestDistributedGrouping, BadIndexSchemaTest, HardAutoCommitTest, TestRawResponseWriter, DateMathParserTest, InfoHandlerTest, UnloadDistributedZkTest, TestMacros, TestLMDirichletSimilarityFactory, DocValuesMissingTest, TestSchemaSimilarityResource, TestConfigSets, TestTrackingShardHandlerFactory, SolrXmlInZkTest, TestAnalyzedSuggestions, ChaosMonkeySafeLeaderTest, TestHashPartitioner, TestSchemaVersionResource, TestReqParamsAPI, SliceStateTest, SegmentsInfoRequestHandlerTest, RAMDirectoryFactoryTest, LukeRequestHandlerTest, DistributedFacetPivotLongTailTest, HdfsCollectionsAPIDistributedZkTest, TestPartialUpdateDeduplication, DistanceUnitsTest, MBeansHandlerTest, TestFiltering, TestInfoStreamLogging, TestCollationFieldDocValues, TestLuceneMatchVersion, TestRandomFaceting, DistributedQueueTest, RecoveryZkTest, OpenCloseCoreStressTest, BasicDistributedZkTest, ReplicationFactorTest]
   [junit4] Completed on J1 in 80.20s, 1 test, 1 error <<< FAILURES!

[...truncated 712 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 479 suites, 1903 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 54 minutes 3 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 216312738 bytes
Compression is 0.0%
Took 42 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2903 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2903/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:34770/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:34770/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([9D5B81451A89ABB3:150FBE9FB475C64B]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:34770 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: 127.0.0.1:34979 failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:34979 failed to respond
	at __randomizedtesting.SeedInfo.seed([9D5B81451A89ABB3:150FBE9FB475C64B]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:34979 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: 127.0.0.1:54113 failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:54113 failed to respond
	at __randomizedtesting.SeedInfo.seed([9D5B81451A89ABB3:150FBE9FB475C64B]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:54113 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9344 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/init-core-data-001
   [junit4]   2> 101702 T157 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 101778 T157 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 101792 T158 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 102196 T157 oasc.ZkTestServer.run start zk server on port:54100
   [junit4]   2> 102393 T157 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 102703 T157 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 102978 T165 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4dcbbdda name:ZooKeeperConnection Watcher:127.0.0.1:54100 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 102978 T157 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 102979 T157 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 102998 T157 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 103192 T157 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 103193 T157 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 103198 T168 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6e891f79 name:ZooKeeperConnection Watcher:127.0.0.1:54100/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 103199 T157 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 103200 T157 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 103227 T157 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 103232 T157 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 103236 T157 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 103239 T157 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 103243 T157 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 103244 T157 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 103250 T157 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 103251 T157 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 103256 T157 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 103256 T157 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 103261 T157 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 103261 T157 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 103265 T157 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 103265 T157 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 103269 T157 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 103270 T157 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 103300 T157 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 103301 T157 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 103305 T157 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 103306 T157 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 103309 T157 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 103310 T157 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 103314 T157 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 103314 T157 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 103318 T157 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 103318 T157 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 105888 T157 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores/collection1
   [junit4]   2> 105895 T157 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 105899 T157 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:54106
   [junit4]   2> 105899 T157 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/tempDir-001/control/data, hostContext=/, hostPort=54105}
   [junit4]   2> 105908 T157 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 105908 T157 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/'
   [junit4]   2> 105978 T157 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/solr.xml
   [junit4]   2> 106074 T157 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores
   [junit4]   2> 106074 T157 oasc.CoreContainer.<init> New CoreContainer 470219515
   [junit4]   2> 106074 T157 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/]
   [junit4]   2> 106075 T157 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/lib
   [junit4]   2> 106076 T157 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/lib).
   [junit4]   2> 106089 T157 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 106089 T157 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 106089 T157 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 106090 T157 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 106090 T157 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 106090 T157 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 106090 T157 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 106091 T157 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 106091 T157 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 106091 T157 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 106092 T157 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 106099 T157 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 106100 T157 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 106100 T157 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 106101 T157 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 106101 T157 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54100/solr
   [junit4]   2> 106158 T157 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 106159 T157 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 106160 T157 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 106162 T180 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@220fe47c name:ZooKeeperConnection Watcher:127.0.0.1:54100 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 106163 T157 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 106163 T157 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 106184 T157 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 106186 T183 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3f2d3e9d name:ZooKeeperConnection Watcher:127.0.0.1:54100/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 106186 T157 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 106211 T157 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 106229 T157 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 106239 T157 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 106244 T157 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 106248 T157 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 106287 T157 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 106290 T157 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 106293 T157 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 106295 T157 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54105_
   [junit4]   2> 106314 T157 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54105_
   [junit4]   2> 106318 T157 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 106322 T157 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 106325 T157 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 106364 T157 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:54105_
   [junit4]   2> 106365 T157 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 106368 T157 oasc.Overseer.start Overseer (id=93610334489739267-127.0.0.1:54105_-n_0000000000) starting
   [junit4]   2> 106410 T157 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 106473 T157 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 106512 T185 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 106512 T157 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 106546 T184 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 106576 T157 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores
   [junit4]   2> 106607 T157 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {schema=schema.xml, coreNodeName=, shard=, name=collection1, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores/collection1, loadOnStartup=true, dataDir=data/, config=solrconfig.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores/collection1/, collection=control_collection}
   [junit4]   2> 106607 T157 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores/collection1/
   [junit4]   2> 106608 T157 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 106609 T187 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 106610 T187 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 106633 T183 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 106633 T187 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 106642 T184 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:54105_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54105",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null} current state version: 0
   [junit4]   2> 106722 T184 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:54105_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54105",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 106733 T184 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 106816 T184 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 106843 T183 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 107634 T187 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 107634 T187 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 107635 T187 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 107635 T187 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 107637 T187 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 107637 T187 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores/collection1/'
   [junit4]   2> 107677 T187 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 107698 T187 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 107728 T187 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 107752 T187 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 107797 T187 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 107799 T187 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 107817 T187 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 108335 T187 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 108376 T187 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 108378 T187 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 108399 T187 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 108424 T187 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 108471 T187 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 108473 T187 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 108473 T187 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 108474 T187 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 108475 T187 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 108475 T187 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 108476 T187 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 108476 T187 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 108476 T187 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores/collection1/data/
   [junit4]   2> 108477 T187 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@44a0c74c
   [junit4]   2> 108488 T187 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores/collection1/data
   [junit4]   2> 108488 T187 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores/collection1/data/index/
   [junit4]   2> 108488 T187 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 108489 T187 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/control-001/cores/collection1/data/index
   [junit4]   2> 108489 T187 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=14, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 108490 T187 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1892ebd6 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55b93b34),segFN=segments_1,generation=1}
   [junit4]   2> 108490 T187 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 108498 T187 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 108605 T187 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 108606 T187 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 108606 T187 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 108606 T187 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 108606 T187 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 108607 T187 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 108607 T187 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 108607 T187 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 108633 T187 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 108643 T187 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 108645 T187 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 108648 T187 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 108659 T187 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update,/admin/segments,standard,/admin/logging,/admin/plugins,/admin/system,/admin/file,/get,/update/json/docs,/admin/luke,/admin/mbeans,/admin/ping,/schema,/replication,/config,/update/json,/admin/properties,/update/csv,/admin/threads
   [junit4]   2> 108663 T187 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 108665 T187 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 108666 T187 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 108667 T187 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 108668 T187 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 108669 T187 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=25, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.792881320471249]
   [junit4]   2> 108669 T187 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1892ebd6 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55b93b34),segFN=segments_1,generation=1}
   [junit4]   2> 108669 T187 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 108670 T187 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@658eb67b[collection1] main
   [junit4]   2> 108670 T187 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 108672 T187 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 108672 T187 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 108690 T187 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 108690 T187 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 108690 T187 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 108691 T187 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 108691 T187 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 108692 T187 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 108692 T187 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 108695 T188 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@658eb67b[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 108705 T187 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 108706 T191 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:54105 collection:control_collection shard:shard1
   [junit4]   2> 108707 T157 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 108708 T157 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 108758 T191 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 108768 T191 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 108771 T183 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 108772 T191 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 108772 T191 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 108773 T184 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C30 name=collection1 org.apache.solr.core.SolrCore@211f7810 url=http://127.0.0.1:54105/collection1 node=127.0.0.1:54105_ C30_STATE=coll:control_collection core:collection1 props:{core=collection1, state=down, node_name=127.0.0.1:54105_, base_url=http://127.0.0.1:54105}
   [junit4]   2> 108786 T191 C:control_collection S:shard1 C30 P54105 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:54105/collection1/
   [junit4]   2> 108787 T191 C:control_collection S:shard1 C30 P54105 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 108787 T191 C:control_collection S:shard1 C30 P54105 oasc.SyncStrategy.syncToMe http://127.0.0.1:54105/collection1/ has no replicas
   [junit4]   2> 108788 T191 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:54105/collection1/ shard1
   [junit4]   2> 108789 T183 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 108807 T157 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 108808 T191 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 108808 T157 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 108818 T195 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@40b5a9b2 name:ZooKeeperConnection Watcher:127.0.0.1:54100/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 108818 T157 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 108819 T157 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 108819 T157 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 108823 T183 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 108825 T184 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54105",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 108827 T157 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 108827 T157 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 108828 T157 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 108828 T157 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 108830 T198 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2f3e04c9 name:ZooKeeperConnection Watcher:127.0.0.1:54100/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 108831 T157 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 108831 T157 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 108834 T183 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 108836 T184 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 108836 T184 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 108837 T195 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 108837 T183 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 108875 T191 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:54105/collection1/ and leader is http://127.0.0.1:54105/collection1/
   [junit4]   2> 108876 T191 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:54105
   [junit4]   2> 108876 T191 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 108876 T191 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 108876 T191 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 108878 T183 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 108880 T184 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:54105_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54105",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "roles":null} current state version: 3
   [junit4]   2> 108881 T184 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:54105_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54105",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 108882 T184 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 108884 T183 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 108884 T195 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 108992 T195 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 108992 T183 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 110958 T157 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores/collection1
   [junit4]   2> 110962 T157 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001
   [junit4]   2> 110963 T157 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 110966 T157 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:54114
   [junit4]   2> 110967 T157 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=54113, solrconfig=solrconfig.xml, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/tempDir-001/jetty1}
   [junit4]   2> 110968 T157 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 110968 T157 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/'
   [junit4]   2> 111057 T157 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/solr.xml
   [junit4]   2> 111148 T157 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores
   [junit4]   2> 111148 T157 oasc.CoreContainer.<init> New CoreContainer 1308631172
   [junit4]   2> 111149 T157 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/]
   [junit4]   2> 111149 T157 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/lib
   [junit4]   2> 111160 T157 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/lib).
   [junit4]   2> 111175 T157 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 111176 T157 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 111176 T157 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 111176 T157 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 111177 T157 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 111177 T157 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 111178 T157 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 111178 T157 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 111178 T157 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 111179 T157 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 111179 T157 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 111191 T157 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 111192 T157 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 111193 T157 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 111193 T157 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 111193 T157 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54100/solr
   [junit4]   2> 111194 T157 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 111194 T157 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 111195 T157 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 111198 T210 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@45150b1b name:ZooKeeperConnection Watcher:127.0.0.1:54100 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 111198 T157 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 111199 T157 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 111203 T157 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 111205 T213 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@30aa7379 name:ZooKeeperConnection Watcher:127.0.0.1:54100/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 111205 T157 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 111215 T157 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 112227 T157 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54113_
   [junit4]   2> 112229 T157 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54113_
   [junit4]   2> 112234 T157 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 112243 T157 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores
   [junit4]   2> 112245 T157 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=collection1, schema=schema.xml, shard=, loadOnStartup=true, dataDir=data/, name=collection1, config=solrconfig.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores/collection1/, coreNodeName=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores/collection1, transient=false}
   [junit4]   2> 112246 T157 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores/collection1/
   [junit4]   2> 112246 T157 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 112250 T214 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 112250 T214 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 112252 T183 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 112254 T214 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 112254 T214 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 112254 T184 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:54113_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54113",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "roles":null} current state version: 5
   [junit4]   2> 112255 T214 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 112255 T184 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:54113_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54113",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 112256 T184 oasco.ReplicaMutator.updateState Collection already exists with numShards=3
   [junit4]   2> 112256 T184 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard3
   [junit4]   2> 112259 T184 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 112264 T214 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 112265 T214 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 112265 T214 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 112265 T214 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 112266 T214 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 112266 T214 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 112267 T214 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 112267 T214 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores/collection1/'
   [junit4]   2> 112285 T214 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 112289 T214 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 112300 T214 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 112338 T214 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 112388 T214 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 112390 T214 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 112401 T214 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 112910 T214 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 112927 T214 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 112929 T214 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 112948 T214 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 112957 T214 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 112979 T214 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 112980 T214 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 112980 T214 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 112981 T214 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 112982 T214 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 112982 T214 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 112982 T214 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 112982 T214 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 112983 T214 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 112983 T214 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@44a0c74c
   [junit4]   2> 112984 T214 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores/collection1/data
   [junit4]   2> 112984 T214 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 112984 T214 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 112985 T214 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9D5B81451A89ABB3-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 112985 T214 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=14, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 112986 T214 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@adbe555 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6aad3504),segFN=segments_1,generation=1}
   [junit4]   2> 112986 T214 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 112996 T214 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 112997 T214 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 112997 T214 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 112997 T214 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 112997 T214 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 112998 T214 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 112998 T214 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 112999 T214 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 112999 T214 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 113013 T214 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 113016 T214 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 113019 T214 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 113021 T214 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 113049 T214 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update,/admin/segments,standard,/admin/logging,/admin/plugins,/admin/system,/admin/file,/get,/update/json/docs,/admin/luke,/admin/mbeans,/admin/ping,/schema,/replication,/config,/update/json,/admin/properties,/update/csv,/admin/threads
   [junit4]   2> 113051 T214 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 113052 T214 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 113053 T214 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 113054 T214 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 113054 T214 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 113056 T214 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=25, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.792881320471249]
   [junit4]   2> 113056 T214 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@adbe555 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6aad3504),segFN=segments_1,generation=1}
   [junit4]   2> 113057 T214 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 113057 T214 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@578a1898[collection1] main
   [junit4]   2> 113057 T214 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 113059 T214 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 113059 T214 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 113060 T214 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 113060 T214 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 113060 T214 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 113061 T214 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 113061 T214 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 113061 T214 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 113062 T214 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 113072 T215 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@578a1898[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 113073 T214 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 113074 T218 C:collection1 S:shard3 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:54113 collection:collection1 shard:shard3
   [junit4]   2> 113075 T157 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 113075 T157 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 113075 T218 C:collection1 S:shard3 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard3/election
   [junit4]   2> 113083 T218 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard3
   [junit4]   2> 113086 T183 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 113087 T218 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 113087 T218 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C31 name=collection1 org.apache.solr.core.SolrCore@729f3241 url=http://127.0.0.1:54113/collection1 node=127.0.0.1:54113_ C31_STATE=coll:collection1 core:collection1 props:{core=collection1, state=down, node_name=127.0.0.1:54113_, base_url=http://127.0.0.1:54113}
   [junit4]   2> 113088 T218 C:collection1 S:shard3 C31 P54113 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:54113/collection1/
   [junit4]   2> 113088 T218 C:collection1 S:shard3 C31 P54113 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 113088 T184 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 113088 T218 C:collection1 S:shard3 C31 P54113 oasc.SyncStrategy.syncToMe http://127.0.0.1:54113/collection1/ has no replicas
   [junit4]   2> 113089 T218 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:54113/collection1/ shard3
   [junit4]   2> 113089 T218 C:collection1 S:shard3 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard3
   [junit4]   2> 113092 T184 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 113093 T213 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 113095 T213 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 113101 T184 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54113",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 113104 T184 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 113105 T213 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 113106 T213 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 113111 T183 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 113151 T218 C:collection1 S:shard3 oasc.ZkController.register We are http://127.0.0.1:54113/collection1/ and leader is http://127.0.0.1:54113/collection1/
   [junit4]   2> 113152 T218 C:collection1 S:shard3 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:54113
   [junit4]   2> 113153 T218 C:collection1 S:shard3 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 113153 T218 C:collection1 S:shard3 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 113153 T218 C:collection1 S:shard3 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 113156 T183 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 113159 T184 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:54113_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54113",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "roles":null} current state version: 5
   [junit4]   2> 113160 T184 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:54113_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54113",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "roles":null}
   [junit4] 

[...truncated too long message...]

0406174556825
   [junit4]   2> 1762047 T3781 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 1762049 T3670 C:control_collection S:shard1 c:collection1 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/fs,null}
   [junit4]   2> 1762225 T3670 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:34962 34962
   [junit4]   2> 1762569 T3671 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:34962 34962
   [junit4]   2> 1762570 T3671 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 		2	/solr/collections/collection1/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1762571 T3670 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:34979/fs
   [junit4]   2> 1762571 T3670 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:34969/fs
   [junit4]   2> 1762571 T3670 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:34986/fs
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=9D5B81451A89ABB3 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=es_ES -Dtests.timezone=Pacific/Midway -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   71.9s J0 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:34979 failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([9D5B81451A89ABB3:150FBE9FB475C64B]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:34979 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 1762587 T3670 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 9D5B81451A89ABB3-001
   [junit4]   2> 71897 T3669 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Lucene50, sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {}, locale=es_ES, timezone=Pacific/Midway
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=185934920,total=312475648
   [junit4]   2> NOTE: All tests run in this JVM: [TestUniqueKeyFieldResource, OutOfBoxZkACLAndCredentialsProvidersTest, SystemInfoHandlerTest, TestReplicaProperties, ZkStateWriterTest, TestHdfsUpdateLog, TestBinaryResponseWriter, TestWriterPerf, FastVectorHighlighterTest, TestLRUCache, TestStressVersions, TestAddFieldRealTimeGet, PrimUtilsTest, MinimalSchemaTest, OverseerCollectionProcessorTest, SuggesterFSTTest, BlockCacheTest, MBeansHandlerTest, PrimitiveFieldTypeTest, TestManagedResource, TestRecovery, HttpPartitionTest, RequestHandlersTest, BadComponentTest, TestJettySolrRunner, CachingDirectoryFactoryTest, TestCloudPivotFacet, CustomCollectionTest, TestNRTOpen, OutputWriterTest, TestConfigSets, TestArbitraryIndexDir, TestFastOutputStream, RecoveryZkTest, TestHighFrequencyDictionaryFactory, BinaryUpdateRequestHandlerTest, TestManagedSchemaFieldTypeResource, TestExactSharedStatsCache, BasicZkTest, TestSort, SolrTestCaseJ4Test, TestDocBasedVersionConstraints, TestSolrQueryParserResource, DefaultValueUpdateProcessorTest, DirectSolrSpellCheckerTest, DeleteLastCustomShardedReplicaTest, ClusterStateUpdateTest, TestRangeQuery, TestExpandComponent, MergeStrategyTest, TestLMDirichletSimilarityFactory, TestSolrDynamicMBean, TestConfig, SimplePostToolTest, TestBulkSchemaConcurrent, TestIntervalFaceting, DistributedQueryComponentOptimizationTest, TestMacros, HdfsUnloadDistributedZkTest, FullSolrCloudDistribCmdsTest, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J0 in 72.68s, 1 test, 1 error <<< FAILURES!

[...truncated 723 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 479 suites, 1903 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 60 minutes 57 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 227544524 bytes
Compression is 0.0%
Took 45 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2902 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2902/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:62732/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:62732/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([26BC33507B4A9FC1:AEE80C8AD5B6F239]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:62732 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: 127.0.0.1:63007 failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:63007 failed to respond
	at __randomizedtesting.SeedInfo.seed([26BC33507B4A9FC1:AEE80C8AD5B6F239]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:63007 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: 127.0.0.1:62804 failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:62804 failed to respond
	at __randomizedtesting.SeedInfo.seed([26BC33507B4A9FC1:AEE80C8AD5B6F239]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:62804 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 10002 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/init-core-data-001
   [junit4]   2> 1486847 T4980 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 1486852 T4980 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1486852 T4981 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1486952 T4980 oasc.ZkTestServer.run start zk server on port:62708
   [junit4]   2> 1486953 T4980 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1486953 T4980 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1486957 T4988 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@37840492 name:ZooKeeperConnection Watcher:127.0.0.1:62708 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1486957 T4980 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1486957 T4980 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1486957 T4980 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1486959 T4980 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1486960 T4980 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1486961 T4991 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@e4dd35c name:ZooKeeperConnection Watcher:127.0.0.1:62708/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1486962 T4980 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1486962 T4980 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1486962 T4980 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1486964 T4980 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1486965 T4980 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1486966 T4980 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1486967 T4980 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1486968 T4980 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1486969 T4980 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1486970 T4980 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1487073 T4980 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1487073 T4980 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1487075 T4980 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1487075 T4980 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1487077 T4980 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1487077 T4980 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1487079 T4980 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1487079 T4980 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1487081 T4980 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1487081 T4980 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1487083 T4980 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1487083 T4980 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1487085 T4980 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1487085 T4980 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1487087 T4980 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1487087 T4980 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1487089 T4980 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1487089 T4980 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1488674 T4980 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores/collection1
   [junit4]   2> 1488678 T4980 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1488682 T4980 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:62717
   [junit4]   2> 1488683 T4980 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, hostPort=62716, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/tempDir-001/control/data}
   [junit4]   2> 1488683 T4980 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 1488683 T4980 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/'
   [junit4]   2> 1488727 T4980 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/solr.xml
   [junit4]   2> 1488781 T4980 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores
   [junit4]   2> 1488782 T4980 oasc.CoreContainer.<init> New CoreContainer 1635486708
   [junit4]   2> 1488782 T4980 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/]
   [junit4]   2> 1488782 T4980 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/lib
   [junit4]   2> 1488783 T4980 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/lib).
   [junit4]   2> 1488792 T4980 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1488793 T4980 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1488793 T4980 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1488793 T4980 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1488793 T4980 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1488794 T4980 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1488794 T4980 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1488794 T4980 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1488794 T4980 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1488794 T4980 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1488795 T4980 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1488802 T4980 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1488803 T4980 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1488803 T4980 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1488803 T4980 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1488804 T4980 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:62708/solr
   [junit4]   2> 1488804 T4980 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1488804 T4980 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1488805 T4980 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1488806 T5003 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7be47528 name:ZooKeeperConnection Watcher:127.0.0.1:62708 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1488807 T4980 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1488807 T4980 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1488809 T4980 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1488810 T5006 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7c304ccf name:ZooKeeperConnection Watcher:127.0.0.1:62708/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1488810 T4980 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1488811 T4980 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1488813 T4980 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1488814 T4980 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1488815 T4980 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1488817 T4980 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1488818 T4980 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1488819 T4980 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1488820 T4980 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1488821 T4980 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:62716_
   [junit4]   2> 1488821 T4980 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:62716_
   [junit4]   2> 1488822 T4980 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1488823 T4980 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1488824 T4980 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1488825 T4980 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:62716_
   [junit4]   2> 1488825 T4980 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1488826 T4980 oasc.Overseer.start Overseer (id=93609472381485059-127.0.0.1:62716_-n_0000000000) starting
   [junit4]   2> 1488827 T4980 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1488831 T4980 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 1488832 T5008 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1488832 T4980 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1488832 T5007 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1488835 T4980 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores
   [junit4]   2> 1488837 T4980 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores/collection1, collection=control_collection, schema=schema.xml, coreNodeName=, shard=, loadOnStartup=true, config=solrconfig.xml, dataDir=data/, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores/collection1/}
   [junit4]   2> 1488837 T4980 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores/collection1/
   [junit4]   2> 1488837 T4980 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1488838 T5010 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1488838 T5010 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1488839 T5010 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1488839 T5006 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1488839 T5007 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62716",
   [junit4]   2> 	  "node_name":"127.0.0.1:62716_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null} current state version: 0
   [junit4]   2> 1488840 T5007 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62716",
   [junit4]   2> 	  "node_name":"127.0.0.1:62716_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 1488840 T5007 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 1488840 T5007 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1488841 T5006 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1489839 T5010 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1489840 T5010 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1489841 T5010 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1489841 T5010 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1489841 T5010 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1489842 T5010 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores/collection1/'
   [junit4]   2> 1489860 T5010 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1489862 T5010 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1489873 T5010 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1489897 T5010 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1489950 T5010 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1490052 T5010 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1490063 T5010 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1490500 T5010 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1490516 T5010 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1490519 T5010 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1490528 T5010 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1490533 T5010 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1490537 T5010 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1490537 T5010 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1490538 T5010 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1490538 T5010 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1490539 T5010 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1490539 T5010 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1490539 T5010 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1490539 T5010 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1490540 T5010 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores/collection1/data/
   [junit4]   2> 1490540 T5010 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@78e9fb08
   [junit4]   2> 1490541 T5010 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores/collection1/data
   [junit4]   2> 1490541 T5010 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores/collection1/data/index/
   [junit4]   2> 1490541 T5010 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1490542 T5010 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/control-001/cores/collection1/data/index
   [junit4]   2> 1490542 T5010 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=36, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1490543 T5010 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@cbce229 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@44a3f847),segFN=segments_1,generation=1}
   [junit4]   2> 1490543 T5010 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1490549 T5010 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1490549 T5010 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1490550 T5010 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1490550 T5010 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1490550 T5010 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1490550 T5010 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1490551 T5010 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1490551 T5010 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1490552 T5010 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1490574 T5010 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1490577 T5010 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1490579 T5010 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1490581 T5010 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1490587 T5010 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: standard,/replication,/admin/system,/admin/properties,/update/json,/admin/file,/update,/admin/logging,/config,/update/json/docs,/admin/plugins,/admin/ping,/admin/threads,/admin/mbeans,/admin/luke,/get,/admin/segments,/schema,/update/csv
   [junit4]   2> 1490589 T5010 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1490590 T5010 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1490591 T5010 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1490592 T5010 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1490592 T5010 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1490593 T5010 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=27, maxMergedSegmentMB=92.4228515625, floorSegmentMB=0.3427734375, forceMergeDeletesPctAllowed=1.1550384131963098, segmentsPerTier=31.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.49791742044625653
   [junit4]   2> 1490594 T5010 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@cbce229 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@44a3f847),segFN=segments_1,generation=1}
   [junit4]   2> 1490594 T5010 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1490594 T5010 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@460fdd6d[collection1] main
   [junit4]   2> 1490595 T5010 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1490595 T5010 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1490596 T5010 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1490596 T5010 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1490596 T5010 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1490596 T5010 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1490597 T5010 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1490597 T5010 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1490597 T5010 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1490598 T5010 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1490598 T5011 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@460fdd6d[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1490599 T5010 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1490600 T5014 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:62716 collection:control_collection shard:shard1
   [junit4]   2> 1490600 T4980 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 1490600 T4980 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1490600 T5014 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1490602 T4980 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1490603 T4980 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1490604 T5014 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1490604 T5018 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@44539bd3 name:ZooKeeperConnection Watcher:127.0.0.1:62708/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1490605 T4980 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1490605 T4980 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1490605 T4980 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1490605 T5006 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1490606 T5014 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1490606 T5014 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C660 name=collection1 org.apache.solr.core.SolrCore@4ecb64b6 url=http://127.0.0.1:62716/collection1 node=127.0.0.1:62716_ C660_STATE=coll:control_collection core:collection1 props:{core=collection1, base_url=http://127.0.0.1:62716, node_name=127.0.0.1:62716_, state=down}
   [junit4]   2> 1490607 T5014 C:control_collection S:shard1 C660 P62716 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:62716/collection1/
   [junit4]   2> 1490607 T5007 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 1490607 T5014 C:control_collection S:shard1 C660 P62716 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1490607 T5014 C:control_collection S:shard1 C660 P62716 oasc.SyncStrategy.syncToMe http://127.0.0.1:62716/collection1/ has no replicas
   [junit4]   2> 1490607 T5014 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:62716/collection1/ shard1
   [junit4]   2> 1490608 T5014 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1490608 T4980 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1490608 T5006 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1490608 T4980 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 1490608 T5018 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1490609 T4980 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1490610 T4980 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1490611 T5021 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2b9870e0 name:ZooKeeperConnection Watcher:127.0.0.1:62708/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1490612 T4980 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1490612 T5006 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1490612 T4980 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1490613 T5007 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62716",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 1490615 T5007 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 1490615 T5007 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 1490616 T5018 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1490616 T5006 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1490618 T5006 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1490618 T5007 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 1490619 T5018 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1490619 T5006 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1490663 T5014 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:62716/collection1/ and leader is http://127.0.0.1:62716/collection1/
   [junit4]   2> 1490664 T5014 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:62716
   [junit4]   2> 1490664 T5014 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1490664 T5014 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1490664 T5014 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1490666 T5006 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1490667 T5007 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62716",
   [junit4]   2> 	  "node_name":"127.0.0.1:62716_",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1"} current state version: 4
   [junit4]   2> 1490667 T5007 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62716",
   [junit4]   2> 	  "node_name":"127.0.0.1:62716_",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1"}
   [junit4]   2> 1490771 T5006 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1490771 T5018 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1492134 T4980 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores/collection1
   [junit4]   2> 1492136 T4980 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001
   [junit4]   2> 1492137 T4980 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1492139 T4980 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:62733
   [junit4]   2> 1492139 T4980 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=62732, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores}
   [junit4]   2> 1492140 T4980 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 1492140 T4980 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/'
   [junit4]   2> 1492176 T4980 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/solr.xml
   [junit4]   2> 1492219 T4980 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores
   [junit4]   2> 1492220 T4980 oasc.CoreContainer.<init> New CoreContainer 125063838
   [junit4]   2> 1492220 T4980 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/]
   [junit4]   2> 1492220 T4980 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/lib
   [junit4]   2> 1492221 T4980 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/lib).
   [junit4]   2> 1492231 T4980 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1492231 T4980 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1492231 T4980 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1492231 T4980 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1492232 T4980 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1492232 T4980 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1492232 T4980 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1492232 T4980 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1492232 T4980 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1492233 T4980 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1492233 T4980 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1492238 T4980 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1492239 T4980 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1492239 T4980 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1492240 T4980 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1492240 T4980 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:62708/solr
   [junit4]   2> 1492240 T4980 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1492240 T4980 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1492241 T4980 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1492243 T5033 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@872ac36 name:ZooKeeperConnection Watcher:127.0.0.1:62708 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1492244 T4980 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1492244 T4980 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1492247 T4980 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1492248 T5036 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@884ee6c name:ZooKeeperConnection Watcher:127.0.0.1:62708/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1492248 T4980 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1492252 T4980 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1493256 T4980 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:62732_
   [junit4]   2> 1493257 T4980 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:62732_
   [junit4]   2> 1493260 T4980 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1493261 T4980 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores
   [junit4]   2> 1493264 T4980 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, coreNodeName=, transient=false, schema=schema.xml, name=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores/collection1/, shard=, dataDir=data/, loadOnStartup=true, collection=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores/collection1}
   [junit4]   2> 1493264 T4980 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores/collection1/
   [junit4]   2> 1493264 T4980 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1493266 T5037 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1493266 T5037 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1493267 T5006 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1493267 T5037 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 1493267 T5037 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 1493267 T5007 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62732",
   [junit4]   2> 	  "node_name":"127.0.0.1:62732_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null} current state version: 5
   [junit4]   2> 1493268 T5037 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 1493268 T5007 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62732",
   [junit4]   2> 	  "node_name":"127.0.0.1:62732_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 1493268 T5007 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 1493268 T5007 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1493268 T5037 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 1493268 T5037 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1493269 T5007 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 1493269 T5036 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1493270 T5036 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 1494269 T5037 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1494270 T5037 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1494270 T5037 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1494270 T5037 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1494271 T5037 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1494271 T5037 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores/collection1/'
   [junit4]   2> 1494284 T5037 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1494286 T5037 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1494294 T5037 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1494324 T5037 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1494362 T5037 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1494464 T5037 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1494472 T5037 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1494762 T5037 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1494774 T5037 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1494776 T5037 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1494784 T5037 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1494787 T5037 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1494790 T5037 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1494791 T5037 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1494791 T5037 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1494791 T5037 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1494792 T5037 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1494792 T5037 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1494792 T5037 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1494793 T5037 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1494793 T5037 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 1494793 T5037 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@78e9fb08
   [junit4]   2> 1494794 T5037 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores/collection1/data
   [junit4]   2> 1494794 T5037 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 1494794 T5037 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1494795 T5037 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 26BC33507B4A9FC1-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 1494795 T5037 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=36, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1494796 T5037 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4d8423ef lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1e5347ab),segFN=segments_1,generation=1}
   [junit4]   2> 1494796 T5037 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1494801 T5037 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1494801 T5037 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1494802 T5037 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1494802 T5037 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1494802 T5037 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1494802 T5037 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1494802 T5037 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1494803 T5037 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1494803 T5037 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1494822 T5037 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1494824 T5037 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1494826 T5037 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1494827 T5037 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1494834 T5037 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: standard,/replication,/admin/system,/admin/properties,/update/json,/admin/file,/update,/admin/logging,/config,/update/json/docs,/admin/plugins,/admin/ping,/admin/threads,/admin/mbeans,/admin/luke,/get,/admin/segments,/schema,/update/csv
   [junit4]   2> 1494836 T5037 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1494838 T5037 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1494838 T5037 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1494839 T5037 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1494839 T5037 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1494840 T5037 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=27, maxMergedSegmentMB=92.4228515625, floorSegmentMB=0.3427734375, forceMergeDeletesPctAllowed=1.1550384131963098, segmentsPerTier=31.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.49791742044625653
   [junit4]   2> 1494841 T5037 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4d8423ef lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1e5347ab),segFN=segments_1,generation=1}
   [junit4]   2> 1494841 T5037 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1494841 T5037 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@56a3f1e1[collection1] main
   [junit4]   2> 1494842 T5037 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1494842 T5037 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1494843 T5037 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1494843 T5037 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1494843 T5037 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1494843 T5037 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1494844 T5037 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1494844 T5037 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1494844 T5037 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1494844 T5037 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1494845 T5038 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@56a3f1e1[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1494845 T5037 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1494846 T5041 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:62732 collection:collection1 shard:shard1
   [junit4]   2> 1494846 T4980 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 1494847 T4980 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1494847 T5041 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1494850 T5041 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1494852 T5006 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1494852 T5041 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1494852 T5041 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C661 name=collection1 org.apache.solr.core.SolrCore@55daa5ce url=http://127.0.0.1:62732/collection1 node=127.0.0.1:62732_ C661_STATE=coll:collection1 core:collection1 props:{core=collection1, base_url=http://127.0.0.1:62732, node_name=127.0.0.1:62732_, state=down}
   [junit4]   2> 1494852 T5041 C:collection1 S:shard1 C661 P62732 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:62732/collection1/
   [junit4]   2> 1494852 T5007 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 1494853 T5041 C:collection1 S:shard1 C661 P62732 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1494853 T5041 C:collection1 S:shard1 C661 P62732 oasc.SyncStrategy.syncToMe http://127.0.0.1:62732/collection1/ has no replicas
   [junit4]   2> 1494853 T5041 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:62732/collection1/ shard1
   [junit4]   2> 1494853 T5041 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1494853 T5007 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 1494854 T5036 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1494855 T5036 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 1494856 T5006 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1494857 T5007 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62732",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 1494858 T5007 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 1494858 T5036 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1494859 T5036 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 1494907 T5041 C:collection1 S:shard1

[...truncated too long message...]

lk to ZK
   [junit4]   2> 1633632 T3227 C:control_collection S:shard1 c:collection1 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> 1633736 T3227 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:62998 62998
   [junit4]   2> 1634082 T3228 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:62998 62998
   [junit4]   2> 1634085 T3228 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 		2	/solr/collections/collection1/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1634085 T3227 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:63011/
   [junit4]   2> 1634086 T3227 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 6 connections to: http://127.0.0.1:63007/
   [junit4]   2> 1634086 T3227 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:63001/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=26BC33507B4A9FC1 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=fr_BE -Dtests.timezone=America/Fortaleza -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   71.9s J0 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:63007 failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([26BC33507B4A9FC1:AEE80C8AD5B6F239]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:63007 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 1634102 T3227 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 26BC33507B4A9FC1-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {timestamp=Lucene50(blocksize=128), a_t=Lucene50(blocksize=128), multiDefault=Lucene50(blocksize=128), id=PostingsFormat(name=LuceneFixedGap), _version_=FSTOrd50, range_facet_l=Lucene50(blocksize=128), intDefault=FSTOrd50, text=FST50}, docValues:{timestamp=DocValuesFormat(name=Direct)}, sim=DefaultSimilarity, locale=fr_BE, timezone=America/Fortaleza
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=97899896,total=327680000
   [junit4]   2> NOTE: All tests run in this JVM: [TestLRUStatsCache, DateRangeFieldTest, LukeRequestHandlerTest, CacheHeaderTest, TestCloudInspectUtil, TestCharFilters, WordBreakSolrSpellCheckerTest, TestJsonRequest, TestPostingsSolrHighlighter, TestBinaryField, TestQueryUtils, TestFieldTypeCollectionResource, TestConfigOverlay, TestFuzzyAnalyzedSuggestions, MultiThreadedOCPTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, DistanceFunctionTest, TestConfig, BlockDirectoryTest, SimpleCollectionCreateDeleteTest, PreAnalyzedUpdateProcessorTest, CursorMarkTest, TestMiniSolrCloudCluster, DeleteLastCustomShardedReplicaTest, AnalyticsQueryTest, SolrInfoMBeanTest, TestCloudManagedSchema, TestDynamicFieldResource, TestIntervalFaceting, AssignTest, HdfsDirectoryFactoryTest, CustomCollectionTest, TestLMDirichletSimilarityFactory, TestLFUCache, DocExpirationUpdateProcessorFactoryTest, HttpPartitionTest, SharedFSAutoReplicaFailoverUtilsTest, ChangedSchemaMergeTest, TestAnalyzedSuggestions, VMParamsZkACLAndCredentialsProvidersTest, TestPHPSerializedResponseWriter, TestFieldCollectionResource, TestInitQParser, HdfsBasicDistributedZkTest, TestZkChroot, CursorPagingTest, RemoteQueryErrorTest, ReturnFieldsTest, ScriptEngineTest, TestCodecSupport, DistributedSuggestComponentTest, TestShardHandlerFactory, BadComponentTest, UpdateRequestProcessorFactoryTest, TestReqParamsAPI, TestImplicitCoreProperties, TestHashPartitioner, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J0 in 71.97s, 1 test, 1 error <<< FAILURES!

[...truncated 722 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 479 suites, 1903 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 54 minutes 45 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 228365171 bytes
Compression is 0.0%
Took 44 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2901 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2901/

4 tests failed.
REGRESSION:  org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds

Error Message:
soft529 wasn't fast enough

Stack Trace:
java.lang.AssertionError: soft529 wasn't fast enough
	at __randomizedtesting.SeedInfo.seed([E74DB75054A0944:5FA022F5B43939E3]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNotNull(Assert.java:526)
	at org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds(SoftAutoCommitTest.java:111)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:25435/_f/em/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:25435/_f/em/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([E74DB75054A0944:8620E4AFABB664BC]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:25435 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: 127.0.0.1:41448 failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:41448 failed to respond
	at __randomizedtesting.SeedInfo.seed([E74DB75054A0944:8620E4AFABB664BC]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:41448 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: 127.0.0.1:43336 failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:43336 failed to respond
	at __randomizedtesting.SeedInfo.seed([E74DB75054A0944:8620E4AFABB664BC]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:43336 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9228 lines...]
   [junit4] Suite: org.apache.solr.update.SoftAutoCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest E74DB75054A0944-001/init-core-data-001
   [junit4]   2> 15934 T16 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 15936 T16 oas.SolrTestCaseJ4.initCore ####initCore
   [junit4]   2> 15936 T16 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 15938 T16 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 15939 T16 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 15940 T16 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 16107 T16 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 16199 T16 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 16345 T16 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 16347 T16 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 16454 T16 oass.IndexSchema.readSchema [null] Schema name=test
   [junit4]   2> 19050 T16 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 19180 T16 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 19190 T16 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 20464 T16 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 20491 T16 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 20497 T16 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 20503 T16 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 20504 T16 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 20504 T16 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 20505 T16 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 20506 T16 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 20519 T16 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 20520 T16 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr
   [junit4]   2> 20520 T16 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/'
   [junit4]   2> 20562 T16 oasc.CoreContainer.<init> New CoreContainer 1713272514
   [junit4]   2> 20563 T16 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/]
   [junit4]   2> 20563 T16 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/lib
   [junit4]   2> 20564 T16 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/lib).
   [junit4]   2> 20581 T16 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 600000
   [junit4]   2> 20582 T16 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 20582 T16 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 60000
   [junit4]   2> 20583 T16 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 20583 T16 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 20584 T16 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 20584 T16 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 20585 T16 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 20585 T16 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 20586 T16 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 20586 T16 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 20595 T16 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=30000&connTimeout=30000&retry=true
   [junit4]   2> 20597 T16 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 20598 T16 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 20598 T16 oasc.CoreContainer.load Node Name: testNode
   [junit4]   2> 20601 T16 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, loadOnStartup=true, config=solrconfig.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, transient=false, collection=collection1, schema=schema.xml, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest E74DB75054A0944-001/init-core-data-001, instanceDir=collection1, shard=shard1}
   [junit4]   2> 20602 T17 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 20604 T17 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 20605 T17 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 20605 T17 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 20720 T17 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 20756 T17 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 20856 T17 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 20858 T17 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 20956 T17 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 21430 T17 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 21456 T17 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 21459 T17 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 21485 T17 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 21491 T17 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 21495 T17 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 21496 T17 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 21497 T17 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 21497 T17 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 21498 T17 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 21498 T17 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 21499 T17 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 21499 T17 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 21500 T17 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest E74DB75054A0944-001/init-core-data-001/
   [junit4]   2> 21508 T17 c:collection1 oasc.JmxMonitoredMap.<init> No JMX servers found, not exposing Solr information with JMX.
   [junit4]   2> 21508 T17 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest E74DB75054A0944-001/init-core-data-001
   [junit4]   2> 21509 T17 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest E74DB75054A0944-001/init-core-data-001/index/
   [junit4]   2> 21509 T17 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest E74DB75054A0944-001/init-core-data-001/index' doesn't exist. Creating new index...
   [junit4]   2> 21510 T17 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest E74DB75054A0944-001/init-core-data-001/index
   [junit4]   2> 21511 T17 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=10, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 21519 T17 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7d97bf8b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a07f17),segFN=segments_1,generation=1}
   [junit4]   2> 21520 T17 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 21856 T17 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 21973 T17 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 21981 T17 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 21982 T17 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 21982 T17 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 21983 T17 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 21983 T17 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 22017 T17 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 22018 T17 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 22335 T17 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 22336 T17 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 22337 T17 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 22374 T17 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 22378 T17 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 22381 T17 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 22384 T17 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 22689 T17 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 22815 T17 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/ping,/update/json,/replication,spellCheckCompRH_Direct,/get,mock,/search-facet-invariants,mltrh,spellCheckWithWordbreak_Direct,/admin/plugins,/update/csv,/admin/file,/admin/mbeans,dismax,/search-facet-def,spellCheckWithWordbreak,defaults,/admin/segments,/admin/threads,/config,tvrh,/admin/properties,standard,/debug/dump,/update/json/docs,lazy,/mlt,spellCheckCompRH1,/schema,/terms,spellCheckCompRH,/admin/system,/admin/logging,/update,/admin/luke
   [junit4]   2> 22827 T17 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 22831 T17 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 22965 T17 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 22966 T17 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 23032 T17 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 23033 T17 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 23034 T17 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=11, maxMergedSegmentMB=98.2822265625, floorSegmentMB=1.3701171875, forceMergeDeletesPctAllowed=26.756904425149433, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8147500519206919
   [junit4]   2> 23035 T17 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7d97bf8b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a07f17),segFN=segments_1,generation=1}
   [junit4]   2> 23036 T17 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 23037 T17 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@785e5b21[collection1] main
   [junit4]   2> 23056 T17 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 23057 T17 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 23058 T17 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 23059 T17 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 23542 T17 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 24040 T17 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 24041 T18 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 24045 T18 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 24045 T18 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 24046 T18 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 24049 T18 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 24052 T18 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 24054 T17 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 24054 T18 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 24056 T18 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 24058 T18 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 24059 T18 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@785e5b21[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 24092 T16 oas.SolrTestCaseJ4.initCore ####initCore end
   [junit4]   2> 24107 T16 oas.SolrTestCaseJ4.setUp ###Starting testSoftAndHardCommitMaxTimeMixedAdds
   [junit4]   2> 24108 T16 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 24111 T16 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 24112 T16 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 24113 T16 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 24359 T16 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 24424 T16 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 24554 T16 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 24555 T16 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 24619 T16 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 25072 T16 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 25097 T16 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 25100 T16 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 25126 T16 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 25139 T16 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 25144 T16 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 25146 T16 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 25146 T16 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 25147 T16 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 25148 T16 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 25148 T16 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 25149 T16 oasc.CoreContainer.reload Reloading SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 25150 T16 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest E74DB75054A0944-001/init-core-data-001/
   [junit4]   2> 25150 T16 c:collection1 oasc.JmxMonitoredMap.<init> No JMX servers found, not exposing Solr information with JMX.
   [junit4]   2> 25151 T16 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest E74DB75054A0944-001/init-core-data-001/index/
   [junit4]   2> 25155 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 25156 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 25156 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 25157 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 25157 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 25158 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 25158 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 25159 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 25159 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 25160 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 25161 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 25161 T16 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 25192 T16 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 25194 T16 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 25196 T16 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 25199 T16 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 25202 T16 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 25204 T16 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/ping,/update/json,/replication,spellCheckCompRH_Direct,/get,mock,/search-facet-invariants,mltrh,spellCheckWithWordbreak_Direct,/admin/plugins,/update/csv,/admin/file,/admin/mbeans,dismax,/search-facet-def,spellCheckWithWordbreak,defaults,/admin/segments,/admin/threads,/config,tvrh,/admin/properties,standard,/debug/dump,/update/json/docs,lazy,/mlt,spellCheckCompRH1,/schema,/terms,spellCheckCompRH,/admin/system,/admin/logging,/update,/admin/luke
   [junit4]   2> 25206 T16 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 25209 T16 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 25217 T16 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 25218 T16 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 25219 T16 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@5ccd6f35[collection1] main
   [junit4]   2> 25219 T16 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 25220 T16 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 25220 T16 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 25221 T16 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 25224 T16 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 25239 T16 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 25240 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 25240 T16 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
   [junit4]   2> 25241 T16 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
   [junit4]   2> 25241 T16 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Closing old IndexWriter... core=collection1
   [junit4]   2> 25241 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 25242 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 25243 T16 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=15, maxMergeAtOnceExplicit=42, maxMergedSegmentMB=76.8359375, floorSegmentMB=1.505859375, forceMergeDeletesPctAllowed=23.071419198270572, segmentsPerTier=37.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 25243 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 25244 T16 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7d97bf8b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a07f17),segFN=segments_1,generation=1}
   [junit4]   2> 25245 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 25245 T16 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 25246 T16 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
   [junit4]   2> 25246 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 25246 T16 c:collection1 oasc.SolrCore.getSearcher WARN [collection1] PERFORMANCE WARNING: Overlapping onDeckSearchers=2
   [junit4]   2> 25247 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 25247 T16 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@760d62e0[collection1] main
   [junit4]   2> 25248 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 25248 T16 c:collection1 oasc.CoreContainer.registerCore replacing core: collection1
   [junit4]   2> 25249 T16 c:collection1 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@744e4cf6
   [junit4]   2> 25250 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 25250 T16 c:collection1 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=0,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=0,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=0,transaction_logs_total_number=0}
   [junit4]   2> 25250 T20 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5ccd6f35[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 25251 T16 c:collection1 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 25251 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 25253 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 25253 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 25253 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 25255 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 25256 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 25257 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 25258 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 25259 T20 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 25260 T20 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@760d62e0[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> ASYNC  NEW_CORE C0 name=collection1 org.apache.solr.core.SolrCore@58169f1b
   [junit4]   2> 26863 T16 c:collection1 C0 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[529 (1497736543998050304)]} 0 1594
   [junit4]   2> 27353 T22 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 27945 T16 c:collection1 oas.SolrTestCaseJ4.tearDown ###Ending testSoftAndHardCommitMaxTimeMixedAdds
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=SoftAutoCommitTest -Dtests.method=testSoftAndHardCommitMaxTimeMixedAdds -Dtests.seed=E74DB75054A0944 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=sr_BA_#Latn -Dtests.timezone=America/Merida -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE 3.92s J2 | SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: soft529 wasn't fast enough
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([E74DB75054A0944:5FA022F5B43939E3]:0)
   [junit4]    > 	at org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds(SoftAutoCommitTest.java:111)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 28025 T16 c:collection1 oas.SolrTestCaseJ4.setUp ###Starting testSoftAndHardCommitMaxTimeDelete
   [junit4]   2> 28026 T16 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 28027 T16 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 28028 T16 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 28029 T16 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 28053 T21 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 28187 T16 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 28231 T16 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 28350 T16 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 28353 T16 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 28361 T22 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@6ec3e67f[collection1] main
   [junit4]   2> 28410 T22 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 28411 T20 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6ec3e67f[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):C1)))}
   [junit4]   2> 28413 T21 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7d97bf8b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a07f17),segFN=segments_1,generation=1}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7d97bf8b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a07f17),segFN=segments_2,generation=2}
   [junit4]   2> 28413 T21 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
   [junit4]   2> 28415 T21 c:collection1 oasc.SolrCore.openNewSearcher SolrIndexSearcher has not changed - not re-opening: org.apache.solr.search.SolrIndexSearcher
   [junit4]   2> 28416 T21 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 28513 T16 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 28954 T16 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 28979 T16 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 28982 T16 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 29008 T16 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 29014 T16 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 29019 T16 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 29020 T16 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 29021 T16 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 29021 T16 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 29022 T16 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 29022 T16 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 29023 T16 c:collection1 oasc.CoreContainer.reload Reloading SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 29024 T16 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest E74DB75054A0944-001/init-core-data-001/
   [junit4]   2> 29024 T16 c:collection1 oasc.JmxMonitoredMap.<init> No JMX servers found, not exposing Solr information with JMX.
   [junit4]   2> 29025 T16 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest E74DB75054A0944-001/init-core-data-001/index/
   [junit4]   2> 29028 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 29029 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 29029 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 29030 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 29030 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 29031 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 29032 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 29033 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 29033 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 29034 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 29035 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 29035 T16 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 29060 T16 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 29064 T16 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 29066 T16 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 29069 T16 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 29073 T16 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 29075 T16 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/ping,/update/json,/replication,spellCheckCompRH_Direct,/get,mock,/search-facet-invariants,mltrh,spellCheckWithWordbreak_Direct,/admin/plugins,/update/csv,/admin/file,/admin/mbeans,dismax,/search-facet-def,spellCheckWithWordbreak,defaults,/admin/segments,/admin/threads,/config,tvrh,/admin/properties,standard,/debug/dump,/update/json/docs,lazy,/mlt,spellCheckCompRH1,/schema,/terms,spellCheckCompRH,/admin/system,/admin/logging,/update,/admin/luke
   [junit4]   2> 29078 T16 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 29080 T16 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 29082 T16 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 29082 T16 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 29084 T16 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@4b3fd89[collection1] main
   [junit4]   2> 29085 T16 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 29086 T16 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 29087 T16 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 29088 T16 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 29092 T16 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 29111 T16 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 29112 T23 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 29112 T16 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
   [junit4]   2> 29113 T16 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
   [junit4]   2> 29114 T16 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Closing old IndexWriter... core=collection1
   [junit4]   2> 29114 T23 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 29114 T23 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 29115 T23 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 29115 T16 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=43, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 29116 T23 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 29118 T23 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 29119 T23 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 29120 T16 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7d97bf8b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a07f17),segFN=segments_2,generation=2}
   [junit4]   2> 29120 T23 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 29120 T16 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
   [junit4]   2> 29121 T16 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
   [junit4]   2> 29121 T23 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 29122 T16 c:collection1 oasc.SolrCore.getSearcher WARN [collection1] PERFORMANCE WARNING: Overlapping onDeckSearchers=2
   [junit4]   2> 29123 T23 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4b3fd89[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):C1)))}
   [junit4]   2> 29128 T16 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@7f3b346b[collection1] main
   [junit4]   2> 29130 T16 c:collection1 oasc.CoreContainer.registerCore replacing core: collection1
   [junit4]   2> 29130 T23 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7f3b346b[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):C1)))}
   [junit4]   2> 29130 T16 c:collection1 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@58169f1b
   [junit4]   2> 29132 T16 c:collection1 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=2,autocommit maxTime=1200ms,autocommits=1,soft autocommit maxTime=500ms,soft autocommits=1,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=1,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=143,transaction_logs_total_number=1}
   [junit4]   2> 29133 T16 c:collection1 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> ASYNC  NEW_CORE C1 name=collection1 org.apache.solr.core.SolrCore@1073c45f
   [junit4]   2> 29141 T16 c:collection1 C1 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[529 (1497736547077718016)]} 0 6
   [junit4]   2> 29144 T16 c:collection1 C1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 29340 T16 c:collection1 C1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7d97bf8b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a07f17),segFN=segments_2,generation=2}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7d97bf8b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a07f17),segFN=segments_3,generation=3}
   [junit4]   2> 29341 T16 c:collection1 C1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
   [junit4]   2> 29342 T16 c:collection1 C1 oass.SolrIndexSearcher.<init> Opening Searcher@61563c31[collection1] main
   [junit4]   2> 29343 T16 c:collection1 C1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 29344 T23 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@61563c31[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_1(5.2.0):C1)))}
   [junit4]   2> 29346 T16 c:collection1 C1 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 203
   [junit4]   2> 29356 T16 c:collection1 C1 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {delete=[529 (-1497736547307356160)]} 0 3
   [junit4]   2> 29858 T25 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 29868 T25 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@4e22d17[collection1] main
   [junit4]   2> 29869 T25 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 29869 T23 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4e22d17[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 29873 T16 c:collection1 C1 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[550 (1497736547848421376)]} 0 3
   [junit4]   2> 30374 T25 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 30389 T25 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@71794a83[collection1] main
   [junit4]   2> 30389 T25 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 30390 T23 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@71794a83[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_2(5.2.0):C1)))}
   [junit4]   2> 30558 T24 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 30560 T24 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7d97bf8b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a07f17),segFN=segments_3,generation=3}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7d97bf8b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a07f17),segFN=segments_4,generation=4}
   [junit4]   2> 30560 T24 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 4
   [junit4]   2> 30561 T24 c:collection1 oasc.SolrCore.openNewSearcher SolrIndexSearcher has not changed - not re-opening: org.apache.solr.search.SolrIndexSearcher
   [junit4]   2> 30562 T24 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 32562 T16 c:collection1 oas.SolrTestCaseJ4.tearDown ###Ending testSoftAndHardCommitMaxTimeDelete
   [junit4]   2> 32571 T16 c:collection1 oas.SolrTestCaseJ4.setUp ###Starting testSoftAndHardCommitMaxTimeRapidAdds
   [junit4]   2> 32572 T16 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 32573 T16 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 32574 T16 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 32574 T16 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 32684 T16 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 32714 T16 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 32827 T16 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 32828 T16 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 32868 T16 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 33504 T16 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 33535 T16 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 33540 T16 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 33574 T16 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 33587 T16 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 33594 T16 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 33596 T16 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 33596 T16 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 33597 T16 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 33598 T16 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 33599 T16 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 33600 T16 c:collection1 oasc.CoreContainer.reload Reloading SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 33601 T16 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest E74DB75054A0944-001/init-core-data-001/
   [junit4]   2> 33601 T16 c:collection1 oasc.JmxMonitoredMap.<init> No JMX servers found, not exposing Solr information with JMX.
   [junit4]   2> 33602 T16 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest E74DB75054A0944-001/init-core-data-001/index/
   [junit4]   2> 33607 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 33608 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 33608 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 33609 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 33610 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 33611 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 33611 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 33612 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 33613 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 33614 T16 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 33615 T16 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 33617 T16 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 33640 T16 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 33643 T16 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 33646 T16 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 33649 T16 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 33654 T16 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 33657 T16 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/ping,/update/json,/replication,spellCheckCompRH_Direct,/get,mock,/search-facet-invariants,mltrh,spellCheckWithWordbreak_Direct,/admin/plugins,/update/csv,/admin/file,/admin/mbeans,dismax,/search-facet-def,spellCheckWithWordbreak,defaults,/admin/segments,/admin/threads,/config,tvrh,/admin/properties,standard,/debug/dump,/update/json/docs,lazy,/mlt,spellCheckCompRH1,/schema,/terms,spellCheckCompRH,/admin/system,/admin/logging,/update,/admin/luke
   [junit4]   2> 33660 T16 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 33662 T16 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 33663 T16 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 33664 T16 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 33666 T16 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@1879cd46[collection1] main
   [junit4]   2> 33667 T16 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 33668 T16 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 33668 T16 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 33669 T16 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 33694 T16 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 33709 T16 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 33711 T26 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 33711 T16 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
   [junit4]   2> 33712 T16 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
   [junit4]   2> 33712 T26 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 33713 T16 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Closing old IndexWriter... core=collection1
   [junit4]   2> 33713 T26 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 33714 T26 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 33716 T26 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 33717 T26 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 33719 T26 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 33720 T16 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@28a0eaa9
   [junit4]   2> 33721 T26 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 33723 T26 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 33723 T26 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1879cd46[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_2(5.2.0):C1)))}
   [junit4]   2> 33727 T16 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7d97bf8b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a07f17),segFN=segments_4,generation=4}
   [junit4]   2> 33728 T16 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 4
   [junit4]   2> 33729 T16 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
   [junit4]   2> 33739 T16 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@7cc8ff30[collection1] main
   [junit4]   2> 33741 T16 c:collection1 oasc.CoreContainer.registerCore replacing core: collection1
   [junit4]   2> 33741 T26 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7cc8ff30[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_2(5.2.0):C1)))}
   [junit4]   2> 33742 T16 c:collection1 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@1073c45f
   [junit4]   2> 33745 T16 c:collection1 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=4,autocommit maxTime=1200ms,autocommits=1,soft autocommit maxTime=500ms,soft autocommits=2,optimizes=0,

[...truncated too long message...]

1 c:collection1 oasc.SocketProxy.close WARN Closing 2 connections to: http://127.0.0.1:63890/brdco/r
   [junit4]   2> 1504344 T2393 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:43349/brdco/r
   [junit4]   2> 1504344 T2393 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:41051/brdco/r
   [junit4]   2> 1504345 T2393 C:control_collection S:shard1 c:collection1 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 1504345 T2393 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:43336/brdco/r
   [junit4]   2> 1504345 T2393 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:63890/brdco/r
   [junit4]   2> 1504345 T2393 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:43349/brdco/r
   [junit4]   2> 1504345 T2393 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:41051/brdco/r
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=E74DB75054A0944 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=nl_NL -Dtests.timezone=Antarctica/McMurdo -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   78.9s J0 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:43336 failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([E74DB75054A0944:8620E4AFABB664BC]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:43336 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 1504360 T2393 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E74DB75054A0944-001
   [junit4]   2> 78855 T2392 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {range_facet_l=Lucene50(blocksize=128), multiDefault=Lucene50(blocksize=128), timestamp=Lucene50(blocksize=128), a_t=Lucene50(blocksize=128), intDefault=PostingsFormat(name=LuceneVarGapDocFreqInterval), _version_=PostingsFormat(name=LuceneVarGapDocFreqInterval), id=PostingsFormat(name=LuceneFixedGap), text=PostingsFormat(name=Memory doPackFST= true)}, docValues:{timestamp=DocValuesFormat(name=Lucene50)}, sim=RandomSimilarityProvider(queryNorm=false,coord=yes): {}, locale=nl_NL, timezone=Antarctica/McMurdo
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=140790216,total=342360064
   [junit4]   2> NOTE: All tests run in this JVM: [EnumFieldTest, TestFaceting, DisMaxRequestHandlerTest, TestSolr4Spatial, TermVectorComponentTest, ExternalFileFieldSortTest, CircularListTest, OpenExchangeRatesOrgProviderTest, RecoveryAfterSoftCommitTest, MultiTermTest, LukeRequestHandlerTest, SaslZkACLProviderTest, JsonLoaderTest, StatelessScriptUpdateProcessorFactoryTest, HdfsUnloadDistributedZkTest, RequiredFieldsTest, TestPostingsSolrHighlighter, RemoteQueryErrorTest, DistributedDebugComponentTest, TestSchemaSimilarityResource, HdfsDirectoryTest, CoreMergeIndexesAdminHandlerTest, DocExpirationUpdateProcessorFactoryTest, TestLMDirichletSimilarityFactory, ShardRoutingCustomTest, TestUpdate, SuggestComponentTest, TestAddFieldRealTimeGet, TestSolrQueryParserDefaultOperatorResource, JSONWriterTest, MultiThreadedOCPTest, TestFuzzyAnalyzedSuggestions, TestClassNameShortening, UUIDUpdateProcessorFallbackTest, TestRebalanceLeaders, SpellCheckComponentTest, TestReload, TestExpandComponent, TestSchemaResource, TestExactSharedStatsCache, CSVRequestHandlerTest, TestAnalyzedSuggestions, TestHashPartitioner, IndexSchemaTest, BasicFunctionalityTest, LoggingHandlerTest, TestStressVersions, IndexBasedSpellCheckerTest, TestNoOpRegenerator, TestSolrJ, PreAnalyzedUpdateProcessorTest, SpellCheckCollatorTest, TestCodecSupport, TestBlobHandler, TestCloudManagedSchema, TestFieldResource, TestIntervalFaceting, BadIndexSchemaTest, LeaderFailoverAfterPartitionTest, TestCloudSchemaless, ReplicationFactorTest]
   [junit4] Completed on J0 in 79.38s, 1 test, 1 error <<< FAILURES!

[...truncated 727 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 479 suites, 1903 tests, 3 errors, 1 failure, 61 ignored (21 assumptions)

Total time: 53 minutes 54 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 223598798 bytes
Compression is 0.0%
Took 41 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



Re: [JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2900 - Still Failing

Posted by Mark Miller <ma...@gmail.com>.
Thats a bad fail. Something has probably changed in recovery code.

 - Mark

On Mon, Apr 6, 2015 at 2:11 PM Apache Jenkins Server <
jenkins@builds.apache.org> wrote:

> Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2900/
>
> 4 tests failed.
> REGRESSION:  org.apache.solr.cloud.RecoveryZkTest.test
>
> Error Message:
> shard1 is not consistent.  Got 1023 from http://127.0.0.1:10050/p_/z/
> collection1lastClient and got 295 from http://127.0.0.1:10059/p_/z/
> collection1
>
> Stack Trace:
> java.lang.AssertionError: shard1 is not consistent.  Got 1023 from
> http://127.0.0.1:10050/p_/z/collection1lastClient and got 295 from
> http://127.0.0.1:10059/p_/z/collection1
>         at __randomizedtesting.SeedInfo.seed([FBFBECE5D5AABA29:
> 73AFD33F7B56D7D1]:0)
>         at org.junit.Assert.fail(Assert.java:93)
>         at org.apache.solr.cloud.RecoveryZkTest.test(
> RecoveryZkTest.java:123)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(
> NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(
> DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(
> RandomizedRunner.java:1627)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(
> RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(
> RandomizedRunner.java:872)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(
> RandomizedRunner.java:886)
>         at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$
> ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
>         at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$
> ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
>         at com.carrotsearch.randomizedtesting.rules.
> SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.
> java:53)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(
> TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(
> AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(
> TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures
> $1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.
> evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$
> StatementRunner.run(ThreadLeakControl.java:365)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.
> forkTimeoutingTask(ThreadLeakControl.java:798)
>         at com.carrotsearch.randomizedtesting.
> ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.
> runSingleTest(RandomizedRunner.java:845)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(
> RandomizedRunner.java:747)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(
> RandomizedRunner.java:781)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(
> RandomizedRunner.java:792)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.
> SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.
> java:53)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(
> AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(
> TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.
> NoShadowingOrOverridesOnMethodsRule$1.evaluate(
> NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.
> NoShadowingOrOverridesOnMethodsRule$1.evaluate(
> NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(
> TestRuleAssertionsRequired.java:54)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.
> evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures
> $1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(
> TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$
> StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
>
>
> FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test
>
> Error Message:
> IOException occured when talking to server at:
> http://127.0.0.1:52540/fmq/gz/c8n_1x3_commits_shard1_replica1
>
> Stack Trace:
> org.apache.solr.client.solrj.SolrServerException: IOException occured
> when talking to server at: http://127.0.0.1:52540/fmq/gz/
> c8n_1x3_commits_shard1_replica1
>         at __randomizedtesting.SeedInfo.seed([FBFBECE5D5AABA29:
> 73AFD33F7B56D7D1]:0)
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.
> executeMethod(HttpSolrClient.java:570)
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(
> HttpSolrClient.java:233)
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(
> HttpSolrClient.java:225)
>         at org.apache.solr.client.solrj.SolrRequest.process(
> SolrRequest.java:135)
>         at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.
> java:483)
>         at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.
> java:464)
>         at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommi
> tTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
>         at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(
> LeaderInitiatedRecoveryOnCommitTest.java:64)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(
> NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(
> DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(
> RandomizedRunner.java:1627)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(
> RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(
> RandomizedRunner.java:872)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(
> RandomizedRunner.java:886)
>         at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$
> ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
>         at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$
> ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
>         at com.carrotsearch.randomizedtesting.rules.
> SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.
> java:53)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(
> TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(
> AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(
> TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures
> $1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.
> evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$
> StatementRunner.run(ThreadLeakControl.java:365)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.
> forkTimeoutingTask(ThreadLeakControl.java:798)
>         at com.carrotsearch.randomizedtesting.
> ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.
> runSingleTest(RandomizedRunner.java:845)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(
> RandomizedRunner.java:747)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(
> RandomizedRunner.java:781)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(
> RandomizedRunner.java:792)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.
> SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.
> java:53)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(
> AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(
> TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.
> NoShadowingOrOverridesOnMethodsRule$1.evaluate(
> NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.
> NoShadowingOrOverridesOnMethodsRule$1.evaluate(
> NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(
> TestRuleAssertionsRequired.java:54)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.
> evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures
> $1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(
> TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$
> StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
> Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:52540
> failed to respond
>         at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(
> DefaultHttpResponseParser.java:143)
>         at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(
> DefaultHttpResponseParser.java:57)
>         at org.apache.http.impl.io.AbstractMessageParser.parse(
> AbstractMessageParser.java:261)
>         at org.apache.http.impl.AbstractHttpClientConnection.
> receiveResponseHeader(AbstractHttpClientConnection.java:283)
>         at org.apache.http.impl.conn.DefaultClientConnection.
> receiveResponseHeader(DefaultClientConnection.java:251)
>         at org.apache.http.impl.conn.ManagedClientConnectionImpl.
> receiveResponseHeader(ManagedClientConnectionImpl.java:197)
>         at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(
> HttpRequestExecutor.java:272)
>         at org.apache.http.protocol.HttpRequestExecutor.execute(
> HttpRequestExecutor.java:124)
>         at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(
> DefaultRequestDirector.java:685)
>         at org.apache.http.impl.client.DefaultRequestDirector.execute(
> DefaultRequestDirector.java:487)
>         at org.apache.http.impl.client.AbstractHttpClient.doExecute(
> AbstractHttpClient.java:882)
>         at org.apache.http.impl.client.CloseableHttpClient.execute(
> CloseableHttpClient.java:82)
>         at org.apache.http.impl.client.CloseableHttpClient.execute(
> CloseableHttpClient.java:107)
>         at org.apache.http.impl.client.CloseableHttpClient.execute(
> CloseableHttpClient.java:55)
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.
> executeMethod(HttpSolrClient.java:462)
>         ... 48 more
>
>
> FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test
>
> Error Message:
> org.apache.solr.client.solrj.SolrServerException: IOException occured
> when talking to server at: http://127.0.0.1:50643/collection1
>
> Stack Trace:
> org.apache.solr.client.solrj.SolrServerException:
> org.apache.solr.client.solrj.SolrServerException: IOException occured
> when talking to server at: http://127.0.0.1:50643/collection1
>         at __randomizedtesting.SeedInfo.seed([FBFBECE5D5AABA29:
> 73AFD33F7B56D7D1]:0)
>         at org.apache.solr.client.solrj.impl.CloudSolrClient.
> directUpdate(CloudSolrClient.java:625)
>         at org.apache.solr.client.solrj.impl.CloudSolrClient.
> sendRequest(CloudSolrClient.java:948)
>         at org.apache.solr.client.solrj.impl.CloudSolrClient.
> requestWithRetryOnStaleState(CloudSolrClient.java:839)
>         at org.apache.solr.client.solrj.impl.CloudSolrClient.request(
> CloudSolrClient.java:782)
>         at org.apache.solr.client.solrj.SolrRequest.process(
> SolrRequest.java:135)
>         at org.apache.solr.client.solrj.SolrClient.add(SolrClient.
> java:174)
>         at org.apache.solr.client.solrj.SolrClient.add(SolrClient.
> java:139)
>         at org.apache.solr.client.solrj.SolrClient.add(SolrClient.
> java:153)
>         at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(
> RecoveryAfterSoftCommitTest.java:88)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(
> NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(
> DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(
> RandomizedRunner.java:1627)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(
> RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(
> RandomizedRunner.java:872)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(
> RandomizedRunner.java:886)
>         at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$
> ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
>         at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$
> ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
>         at com.carrotsearch.randomizedtesting.rules.
> SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.
> java:53)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(
> TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(
> AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(
> TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures
> $1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.
> evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$
> StatementRunner.run(ThreadLeakControl.java:365)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.
> forkTimeoutingTask(ThreadLeakControl.java:798)
>         at com.carrotsearch.randomizedtesting.
> ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.
> runSingleTest(RandomizedRunner.java:845)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(
> RandomizedRunner.java:747)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(
> RandomizedRunner.java:781)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(
> RandomizedRunner.java:792)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.
> SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.
> java:53)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(
> AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(
> TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.
> NoShadowingOrOverridesOnMethodsRule$1.evaluate(
> NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.
> NoShadowingOrOverridesOnMethodsRule$1.evaluate(
> NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(
> TestRuleAssertionsRequired.java:54)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.
> evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures
> $1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(
> TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$
> StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
> Caused by: org.apache.solr.client.solrj.SolrServerException: IOException
> occured when talking to server at: http://127.0.0.1:50643/collection1
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.
> executeMethod(HttpSolrClient.java:570)
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(
> HttpSolrClient.java:233)
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(
> HttpSolrClient.java:225)
>         at org.apache.solr.client.solrj.impl.LBHttpSolrClient.
> doRequest(LBHttpSolrClient.java:370)
>         at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(
> LBHttpSolrClient.java:325)
>         at org.apache.solr.client.solrj.impl.CloudSolrClient.
> directUpdate(CloudSolrClient.java:622)
>         ... 49 more
> Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:50643
> failed to respond
>         at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(
> DefaultHttpResponseParser.java:143)
>         at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(
> DefaultHttpResponseParser.java:57)
>         at org.apache.http.impl.io.AbstractMessageParser.parse(
> AbstractMessageParser.java:261)
>         at org.apache.http.impl.AbstractHttpClientConnection.
> receiveResponseHeader(AbstractHttpClientConnection.java:283)
>         at org.apache.http.impl.conn.DefaultClientConnection.
> receiveResponseHeader(DefaultClientConnection.java:251)
>         at org.apache.http.impl.conn.ManagedClientConnectionImpl.
> receiveResponseHeader(ManagedClientConnectionImpl.java:197)
>         at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(
> HttpRequestExecutor.java:272)
>         at org.apache.http.protocol.HttpRequestExecutor.execute(
> HttpRequestExecutor.java:124)
>         at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(
> DefaultRequestDirector.java:685)
>         at org.apache.http.impl.client.DefaultRequestDirector.execute(
> DefaultRequestDirector.java:487)
>         at org.apache.http.impl.client.AbstractHttpClient.doExecute(
> AbstractHttpClient.java:882)
>         at org.apache.http.impl.client.CloseableHttpClient.execute(
> CloseableHttpClient.java:82)
>         at org.apache.http.impl.client.CloseableHttpClient.execute(
> CloseableHttpClient.java:107)
>         at org.apache.http.impl.client.CloseableHttpClient.execute(
> CloseableHttpClient.java:55)
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.
> executeMethod(HttpSolrClient.java:462)
>         ... 54 more
>
>
> FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test
>
> Error Message:
> org.apache.solr.client.solrj.SolrServerException: IOException occured
> when talking to server at: http://127.0.0.1:11223/
> repfacttest_c8n_1x3_shard1_replica2
>
> Stack Trace:
> org.apache.solr.client.solrj.SolrServerException:
> org.apache.solr.client.solrj.SolrServerException: IOException occured
> when talking to server at: http://127.0.0.1:11223/
> repfacttest_c8n_1x3_shard1_replica2
>         at __randomizedtesting.SeedInfo.seed([FBFBECE5D5AABA29:
> 73AFD33F7B56D7D1]:0)
>         at org.apache.solr.client.solrj.impl.CloudSolrClient.
> directUpdate(CloudSolrClient.java:625)
>         at org.apache.solr.client.solrj.impl.CloudSolrClient.
> sendRequest(CloudSolrClient.java:948)
>         at org.apache.solr.client.solrj.impl.CloudSolrClient.
> requestWithRetryOnStaleState(CloudSolrClient.java:839)
>         at org.apache.solr.client.solrj.impl.CloudSolrClient.request(
> CloudSolrClient.java:782)
>         at org.apache.solr.client.solrj.SolrClient.request(SolrClient.
> java:1220)
>         at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(
> ReplicationFactorTest.java:308)
>         at org.apache.solr.cloud.ReplicationFactorTest.testRf3(
> ReplicationFactorTest.java:234)
>         at org.apache.solr.cloud.ReplicationFactorTest.test(
> ReplicationFactorTest.java:113)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(
> NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(
> DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(
> RandomizedRunner.java:1627)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(
> RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(
> RandomizedRunner.java:872)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(
> RandomizedRunner.java:886)
>         at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$
> ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
>         at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$
> ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
>         at com.carrotsearch.randomizedtesting.rules.
> SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.
> java:53)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(
> TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(
> AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(
> TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures
> $1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.
> evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$
> StatementRunner.run(ThreadLeakControl.java:365)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.
> forkTimeoutingTask(ThreadLeakControl.java:798)
>         at com.carrotsearch.randomizedtesting.
> ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.
> runSingleTest(RandomizedRunner.java:845)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(
> RandomizedRunner.java:747)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(
> RandomizedRunner.java:781)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(
> RandomizedRunner.java:792)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.
> SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.
> java:53)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(
> AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(
> TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.
> NoShadowingOrOverridesOnMethodsRule$1.evaluate(
> NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.
> NoShadowingOrOverridesOnMethodsRule$1.evaluate(
> NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(
> TestRuleAssertionsRequired.java:54)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.
> evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures
> $1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(
> TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.
> StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$
> StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
> Caused by: org.apache.solr.client.solrj.SolrServerException: IOException
> occured when talking to server at: http://127.0.0.1:11223/
> repfacttest_c8n_1x3_shard1_replica2
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.
> executeMethod(HttpSolrClient.java:570)
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(
> HttpSolrClient.java:233)
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.request(
> HttpSolrClient.java:225)
>         at org.apache.solr.client.solrj.impl.LBHttpSolrClient.
> doRequest(LBHttpSolrClient.java:370)
>         at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(
> LBHttpSolrClient.java:325)
>         at org.apache.solr.client.solrj.impl.CloudSolrClient.
> directUpdate(CloudSolrClient.java:622)
>         ... 48 more
> Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:11223
> failed to respond
>         at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(
> DefaultHttpResponseParser.java:143)
>         at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(
> DefaultHttpResponseParser.java:57)
>         at org.apache.http.impl.io.AbstractMessageParser.parse(
> AbstractMessageParser.java:261)
>         at org.apache.http.impl.AbstractHttpClientConnection.
> receiveResponseHeader(AbstractHttpClientConnection.java:283)
>         at org.apache.http.impl.conn.DefaultClientConnection.
> receiveResponseHeader(DefaultClientConnection.java:251)
>         at org.apache.http.impl.conn.ManagedClientConnectionImpl.
> receiveResponseHeader(ManagedClientConnectionImpl.java:197)
>         at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(
> HttpRequestExecutor.java:272)
>         at org.apache.http.protocol.HttpRequestExecutor.execute(
> HttpRequestExecutor.java:124)
>         at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(
> DefaultRequestDirector.java:685)
>         at org.apache.http.impl.client.DefaultRequestDirector.execute(
> DefaultRequestDirector.java:487)
>         at org.apache.http.impl.client.AbstractHttpClient.doExecute(
> AbstractHttpClient.java:882)
>         at org.apache.http.impl.client.CloseableHttpClient.execute(
> CloseableHttpClient.java:82)
>         at org.apache.http.impl.client.CloseableHttpClient.execute(
> CloseableHttpClient.java:107)
>         at org.apache.http.impl.client.CloseableHttpClient.execute(
> CloseableHttpClient.java:55)
>         at org.apache.solr.client.solrj.impl.HttpSolrClient.
> executeMethod(HttpSolrClient.java:462)
>         ... 53 more
>
>
>
>
> Build Log:
> [...truncated 9494 lines...]
>    [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
>    [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-
> slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/
> solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/init-core-data-001
>    [junit4]   2> 295434 T1269 oas.BaseDistributedSearchTestCase.initHostContext
> Setting hostContext system property: /
>    [junit4]   2> 295441 T1269 oasc.ZkTestServer.run STARTING ZK TEST SERVER
>    [junit4]   1> client port:0.0.0.0/0.0.0.0:0
>    [junit4]   2> 295441 T1270 oasc.ZkTestServer$ZKServerMain.runFromConfig
> Starting server
>    [junit4]   2> 295541 T1269 oasc.ZkTestServer.run start zk server on
> port:50623
>    [junit4]   2> 295542 T1269 oascc.SolrZkClient.
> createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
>    [junit4]   2> 295542 T1269 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>    [junit4]   2> 295545 T1277 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@6e5605f4
> name:ZooKeeperConnection Watcher:127.0.0.1:50623 got event WatchedEvent
> state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 295545 T1269 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>    [junit4]   2> 295546 T1269 oascc.SolrZkClient.createZkACLProvider
> Using default ZkACLProvider
>    [junit4]   2> 295546 T1269 oascc.SolrZkClient.makePath makePath: /solr
>    [junit4]   2> 295548 T1269 oascc.SolrZkClient.
> createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
>    [junit4]   2> 295549 T1269 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>    [junit4]   2> 295550 T1280 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@842caa6
> name:ZooKeeperConnection Watcher:127.0.0.1:50623/solr got event
> WatchedEvent state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 295550 T1269 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>    [junit4]   2> 295551 T1269 oascc.SolrZkClient.createZkACLProvider
> Using default ZkACLProvider
>    [junit4]   2> 295551 T1269 oascc.SolrZkClient.makePath makePath:
> /collections/collection1
>    [junit4]   2> 295553 T1269 oascc.SolrZkClient.makePath makePath:
> /collections/collection1/shards
>    [junit4]   2> 295554 T1269 oascc.SolrZkClient.makePath makePath:
> /collections/control_collection
>    [junit4]   2> 295555 T1269 oascc.SolrZkClient.makePath makePath:
> /collections/control_collection/shards
>    [junit4]   2> 295556 T1269 oasc.AbstractZkTestCase.putConfig put
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml
> to /configs/conf1/solrconfig.xml
>    [junit4]   2> 295557 T1269 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/solrconfig.xml
>    [junit4]   2> 295559 T1269 oasc.AbstractZkTestCase.putConfig put
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
> to /configs/conf1/schema.xml
>    [junit4]   2> 295560 T1269 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/schema.xml
>    [junit4]   2> 295661 T1269 oasc.AbstractZkTestCase.putConfig put
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
> conf/solrconfig.snippet.randomindexconfig.xml to
> /configs/conf1/solrconfig.snippet.randomindexconfig.xml
>    [junit4]   2> 295662 T1269 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/solrconfig.snippet.randomindexconfig.xml
>    [junit4]   2> 295663 T1269 oasc.AbstractZkTestCase.putConfig put
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt
> to /configs/conf1/stopwords.txt
>    [junit4]   2> 295663 T1269 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/stopwords.txt
>    [junit4]   2> 295665 T1269 oasc.AbstractZkTestCase.putConfig put
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt
> to /configs/conf1/protwords.txt
>    [junit4]   2> 295665 T1269 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/protwords.txt
>    [junit4]   2> 295667 T1269 oasc.AbstractZkTestCase.putConfig put
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml
> to /configs/conf1/currency.xml
>    [junit4]   2> 295667 T1269 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/currency.xml
>    [junit4]   2> 295668 T1269 oasc.AbstractZkTestCase.putConfig put
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml
> to /configs/conf1/enumsConfig.xml
>    [junit4]   2> 295669 T1269 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/enumsConfig.xml
>    [junit4]   2> 295670 T1269 oasc.AbstractZkTestCase.putConfig put
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json
> to /configs/conf1/open-exchange-rates.json
>    [junit4]   2> 295670 T1269 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/open-exchange-rates.json
>    [junit4]   2> 295672 T1269 oasc.AbstractZkTestCase.putConfig put
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
> conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-
> ISOLatin1Accent.txt
>    [junit4]   2> 295672 T1269 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/mapping-ISOLatin1Accent.txt
>    [junit4]   2> 295673 T1269 oasc.AbstractZkTestCase.putConfig put
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt
> to /configs/conf1/old_synonyms.txt
>    [junit4]   2> 295674 T1269 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/old_synonyms.txt
>    [junit4]   2> 295675 T1269 oasc.AbstractZkTestCase.putConfig put
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt
> to /configs/conf1/synonyms.txt
>    [junit4]   2> 295675 T1269 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/synonyms.txt
>    [junit4]   2> 296963 T1269 oas.SolrTestCaseJ4.writeCoreProperties
> Writing core.properties file to /usr/home/jenkins/jenkins-
> slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/
> solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores/collection1
>    [junit4]   2> 296967 T1269 oejs.Server.doStart jetty-8.1.10.v20130312
>    [junit4]   2> 296971 T1269 oejs.AbstractConnector.doStart Started
> SocketConnector@127.0.0.1:50636
>    [junit4]   2> 296971 T1269 oascse.JettySolrRunner$1.lifeCycleStarted
> Jetty properties: {hostPort=50635, coreRootDirectory=/usr/home/
> jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.
> x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores, hostContext=/,
> solr.data.dir=/usr/home/jenkins/jenkins-slave/
> workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/
> test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/tempDir-001/control/data}
>    [junit4]   2> 296971 T1269 oass.SolrDispatchFilter.init
> SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
>    [junit4]   2> 296972 T1269 oasc.SolrResourceLoader.<init> new
> SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-
> slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/
> solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/'
>    [junit4]   2> 297015 T1269 oasc.SolrXmlConfig.fromFile Loading
> container configuration from /usr/home/jenkins/jenkins-
> slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/
> solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/solr.xml
>    [junit4]   2> 297067 T1269 oasc.CorePropertiesLocator.<init>
> Config-defined core root directory: /usr/home/jenkins/jenkins-
> slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/
> solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores
>    [junit4]   2> 297069 T1269 oasc.CoreContainer.<init> New CoreContainer
> 2110879613
>    [junit4]   2> 297069 T1269 oasc.CoreContainer.load Loading cores into
> CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/
> workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/
> test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/]
>    [junit4]   2> 297069 T1269 oasc.CoreContainer.load loading shared
> library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/lib
>    [junit4]   2> 297070 T1269 oasc.SolrResourceLoader.addToClassLoader
> WARN Can't find (or read) directory to add to classloader: lib (resolved
> as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/lib).
>    [junit4]   2> 297080 T1269 oashc.HttpShardHandlerFactory.getParameter
> Setting socketTimeout to: 90000
>    [junit4]   2> 297080 T1269 oashc.HttpShardHandlerFactory.getParameter
> Setting urlScheme to:
>    [junit4]   2> 297080 T1269 oashc.HttpShardHandlerFactory.getParameter
> Setting connTimeout to: 15000
>    [junit4]   2> 297080 T1269 oashc.HttpShardHandlerFactory.getParameter
> Setting maxConnectionsPerHost to: 20
>    [junit4]   2> 297081 T1269 oashc.HttpShardHandlerFactory.getParameter
> Setting maxConnections to: 10000
>    [junit4]   2> 297081 T1269 oashc.HttpShardHandlerFactory.getParameter
> Setting corePoolSize to: 0
>    [junit4]   2> 297081 T1269 oashc.HttpShardHandlerFactory.getParameter
> Setting maximumPoolSize to: 2147483647
>    [junit4]   2> 297081 T1269 oashc.HttpShardHandlerFactory.getParameter
> Setting maxThreadIdleTime to: 5
>    [junit4]   2> 297082 T1269 oashc.HttpShardHandlerFactory.getParameter
> Setting sizeOfQueue to: -1
>    [junit4]   2> 297082 T1269 oashc.HttpShardHandlerFactory.getParameter
> Setting fairnessPolicy to: false
>    [junit4]   2> 297082 T1269 oashc.HttpShardHandlerFactory.getParameter
> Setting useRetries to: false
>    [junit4]   2> 297089 T1269 oasu.UpdateShardHandler.<init> Creating
> UpdateShardHandler HTTP client with params: socketTimeout=340000&
> connTimeout=45000&retry=true
>    [junit4]   2> 297090 T1269 oasl.LogWatcher.createWatcher SLF4J impl is
> org.slf4j.impl.Log4jLoggerFactory
>    [junit4]   2> 297090 T1269 oasl.LogWatcher.newRegisteredLogWatcher
> Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>    [junit4]   2> 297090 T1269 oasc.CoreContainer.load Node Name: 127.0.0.1
>    [junit4]   2> 297091 T1269 oasc.ZkContainer.initZooKeeper Zookeeper
> client=127.0.0.1:50623/solr
>    [junit4]   2> 297091 T1269 oasc.ZkController.checkChrootPath zkHost
> includes chroot
>    [junit4]   2> 297092 T1269 oascc.SolrZkClient.
> createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
>    [junit4]   2> 297092 T1269 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>    [junit4]   2> 297094 T1292 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@45cc06e8
> name:ZooKeeperConnection Watcher:127.0.0.1:50623 got event WatchedEvent
> state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 297095 T1269 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>    [junit4]   2> 297096 T1269 oascc.SolrZkClient.createZkACLProvider
> Using default ZkACLProvider
>    [junit4]   2> 297097 T1269 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>    [junit4]   2> 297098 T1295 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@34504d13
> name:ZooKeeperConnection Watcher:127.0.0.1:50623/solr got event
> WatchedEvent state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 297098 T1269 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>    [junit4]   2> 297100 T1269 oascc.SolrZkClient.makePath makePath:
> /overseer/queue
>    [junit4]   2> 297101 T1269 oascc.SolrZkClient.makePath makePath:
> /overseer/collection-queue-work
>    [junit4]   2> 297103 T1269 oascc.SolrZkClient.makePath makePath:
> /overseer/collection-map-running
>    [junit4]   2> 297104 T1269 oascc.SolrZkClient.makePath makePath:
> /overseer/collection-map-completed
>    [junit4]   2> 297105 T1269 oascc.SolrZkClient.makePath makePath:
> /overseer/collection-map-failure
>    [junit4]   2> 297107 T1269 oascc.SolrZkClient.makePath makePath:
> /live_nodes
>    [junit4]   2> 297108 T1269 oascc.SolrZkClient.makePath makePath:
> /aliases.json
>    [junit4]   2> 297109 T1269 oascc.SolrZkClient.makePath makePath:
> /clusterstate.json
>    [junit4]   2> 297110 T1269 oasc.ZkController.createEphemeralLiveNode
> Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50635_
>    [junit4]   2> 297110 T1269 oascc.SolrZkClient.makePath makePath:
> /live_nodes/127.0.0.1:50635_
>    [junit4]   2> 297112 T1269 oascc.SolrZkClient.makePath makePath:
> /overseer_elect
>    [junit4]   2> 297113 T1269 oascc.SolrZkClient.makePath makePath:
> /overseer_elect/election
>    [junit4]   2> 297114 T1269 oasc.Overseer.close Overseer (id=null)
> closing
>    [junit4]   2> 297115 T1269 oasc.OverseerElectionContext.runLeaderProcess
> I am going to be the leader 127.0.0.1:50635_
>    [junit4]   2> 297115 T1269 oascc.SolrZkClient.makePath makePath:
> /overseer_elect/leader
>    [junit4]   2> 297116 T1269 oasc.Overseer.start Overseer
> (id=93607700461191171-127.0.0.1:50635_-n_0000000000) starting
>    [junit4]   2> 297118 T1269 oascc.SolrZkClient.makePath makePath:
> /overseer/queue-work
>    [junit4]   2> 297122 T1269 oasc.OverseerAutoReplicaFailoverThread.<init>
> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000
> autoReplicaFailoverWaitAfterExpiration=30000
> autoReplicaFailoverBadNodeExpiration=60000
>    [junit4]   2> 297122 T1297 oasc.OverseerCollectionProcessor.run
> Process current queue of collection creations
>    [junit4]   2> 297122 T1269 oascc.ZkStateReader.
> createClusterStateWatchersAndUpdate Updating cluster state from
> ZooKeeper...
>    [junit4]   2> 297122 T1296 oasc.Overseer$ClusterStateUpdater.run
> Starting to work on the main queue
>    [junit4]   2> 297126 T1269 oasc.CorePropertiesLocator.discover Looking
> for core definitions underneath /usr/home/jenkins/jenkins-
> slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/
> solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores
>    [junit4]   2> 297128 T1269 oasc.CoreDescriptor.<init> CORE DESCRIPTOR:
> {shard=, collection=control_collection, coreNodeName=, dataDir=data/,
> absoluteInstDir=/usr/home/jenkins/jenkins-slave/
> workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/
> test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores/collection1/,
> config=solrconfig.xml, schema=schema.xml, loadOnStartup=true,
> transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/
> Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores/collection1, name=collection1}
>    [junit4]   2> 297128 T1269 oasc.CorePropertiesLocator.discoverUnder
> Found core collection1 in /usr/home/jenkins/jenkins-
> slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/
> solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores/collection1/
>    [junit4]   2> 297128 T1269 oasc.CorePropertiesLocator.discover Found 1
> core definitions
>    [junit4]   2> 297130 T1299 C:control_collection c:collection1
> oasc.ZkController.publish publishing core=collection1 state=down
> collection=control_collection
>    [junit4]   2> 297130 T1299 C:control_collection c:collection1
> oasc.ZkController.publish numShards not found on descriptor - reading it
> from system property
>    [junit4]   2> 297131 T1299 C:control_collection c:collection1
> oasc.ZkController.waitForCoreNodeName look for our core node name
>    [junit4]   2> 297131 T1295 oasc.DistributedQueue$LatchWatcher.process
> NodeChildrenChanged fired on path /overseer/queue state SyncConnected
>    [junit4]   2> 297132 T1296 oasc.Overseer$ClusterStateUpdater.run
> processMessage: queueSize: 1, message = {
>    [junit4]   2>          "collection":"control_collection",
>    [junit4]   2>          "state":"down",
>    [junit4]   2>          "shard":null,
>    [junit4]   2>          "base_url":"http://127.0.0.1:50635",
>    [junit4]   2>          "roles":null,
>    [junit4]   2>          "node_name":"127.0.0.1:50635_",
>    [junit4]   2>          "core":"collection1",
>    [junit4]   2>          "operation":"state",
>    [junit4]   2>          "numShards":"1"} current state version: 0
>    [junit4]   2> 297132 T1296 oasco.ReplicaMutator.updateState Update
> state numShards=1 message={
>    [junit4]   2>          "collection":"control_collection",
>    [junit4]   2>          "state":"down",
>    [junit4]   2>          "shard":null,
>    [junit4]   2>          "base_url":"http://127.0.0.1:50635",
>    [junit4]   2>          "roles":null,
>    [junit4]   2>          "node_name":"127.0.0.1:50635_",
>    [junit4]   2>          "core":"collection1",
>    [junit4]   2>          "operation":"state",
>    [junit4]   2>          "numShards":"1"}
>    [junit4]   2> 297132 T1296 oasco.ClusterStateMutator.createCollection
> building a new cName: control_collection
>    [junit4]   2> 297133 T1296 oasco.ReplicaMutator.updateState Assigning
> new node to shard shard=shard1
>    [junit4]   2> 297134 T1295 oascc.ZkStateReader$2.process A cluster
> state change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 298131 T1299 C:control_collection c:collection1
> oasc.ZkController.waitForShardId waiting to find shard id in clusterstate
> for collection1
>    [junit4]   2> 298132 T1299 C:control_collection c:collection1
> oasc.ZkController.createCollectionZkNode Check for collection
> zkNode:control_collection
>    [junit4]   2> 298132 T1299 C:control_collection c:collection1
> oasc.ZkController.createCollectionZkNode Collection zkNode exists
>    [junit4]   2> 298132 T1299 C:control_collection c:collection1
> oascc.ZkStateReader.readConfigName Load collection config
> from:/collections/control_collection
>    [junit4]   2> 298133 T1299 C:control_collection c:collection1
> oascc.ZkStateReader.readConfigName path=/collections/control_collection
> configName=conf1 specified config exists in ZooKeeper
>    [junit4]   2> 298133 T1299 C:control_collection c:collection1
> oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory:
> '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores/collection1/'
>    [junit4]   2> 298147 T1299 C:control_collection c:collection1
> oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
>    [junit4]   2> 298149 T1299 C:control_collection c:collection1
> oasc.Config.<init> loaded config solrconfig.xml with version 0
>    [junit4]   2> 298156 T1299 C:control_collection c:collection1
> oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
>    [junit4]   2> 298172 T1299 C:control_collection c:collection1
> oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
>    [junit4]   2> 298210 T1299 C:control_collection c:collection1
> oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
>    [junit4]   2> 298311 T1299 C:control_collection c:collection1
> oass.IndexSchema.readSchema Reading Solr Schema from
> /configs/conf1/schema.xml
>    [junit4]   2> 298318 T1299 C:control_collection c:collection1
> oass.IndexSchema.readSchema [collection1] Schema name=test
>    [junit4]   2> 298623 T1299 C:control_collection c:collection1 oass.
> OpenExchangeRatesOrgProvider.init Initialized with
> rates=open-exchange-rates.json, refreshInterval=1440.
>    [junit4]   2> 298636 T1299 C:control_collection c:collection1
> oass.IndexSchema.readSchema default search field in schema is text
>    [junit4]   2> 298638 T1299 C:control_collection c:collection1
> oass.IndexSchema.readSchema unique key field: id
>    [junit4]   2> 298648 T1299 C:control_collection c:collection1
> oass.FileExchangeRateProvider.reload Reloading exchange rates from file
> currency.xml
>    [junit4]   2> 298651 T1299 C:control_collection c:collection1
> oass.FileExchangeRateProvider.reload Reloading exchange rates from file
> currency.xml
>    [junit4]   2> 298654 T1299 C:control_collection c:collection1 oass.
> OpenExchangeRatesOrgProvider.reload Reloading exchange rates from
> open-exchange-rates.json
>    [junit4]   2> 298655 T1299 C:control_collection c:collection1 oass.
> OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key
> IMPORTANT NOTE
>    [junit4]   2> 298655 T1299 C:control_collection c:collection1 oass.
> OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key,
> got STRING
>    [junit4]   2> 298655 T1299 C:control_collection c:collection1 oass.
> OpenExchangeRatesOrgProvider.reload Reloading exchange rates from
> open-exchange-rates.json
>    [junit4]   2> 298656 T1299 C:control_collection c:collection1 oass.
> OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key
> IMPORTANT NOTE
>    [junit4]   2> 298656 T1299 C:control_collection c:collection1 oass.
> OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key,
> got STRING
>    [junit4]   2> 298656 T1299 C:control_collection c:collection1
> oasc.CoreContainer.create Creating SolrCore 'collection1' using
> configuration from collection control_collection
>    [junit4]   2> 298656 T1299 C:control_collection c:collection1
> oasc.SolrCore.initDirectoryFactory org.apache.solr.core.
> MockDirectoryFactory
>    [junit4]   2> 298657 T1299 C:control_collection c:collection1
> oasc.SolrCore.<init> [collection1] Opening new SolrCore at
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores/collection1/,
> dataDir=/usr/home/jenkins/jenkins-slave/workspace/
> Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores/collection1/data/
>    [junit4]   2> 298657 T1299 C:control_collection c:collection1
> oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans
> to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2ec2b6ef
>    [junit4]   2> 298658 T1299 C:control_collection c:collection1
> oasc.CachingDirectoryFactory.get return new directory for
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores/collection1/data
>    [junit4]   2> 298658 T1299 C:control_collection c:collection1
> oasc.SolrCore.getNewIndexDir New index directory detected: old=null
> new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores/collection1/data/index/
>    [junit4]   2> 298658 T1299 C:control_collection c:collection1
> oasc.SolrCore.initIndex WARN [collection1] Solr index directory
> '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores/collection1/data/index' doesn't
> exist. Creating new index...
>    [junit4]   2> 298658 T1299 C:control_collection c:collection1
> oasc.CachingDirectoryFactory.get return new directory for
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/control-001/cores/collection1/data/index
>    [junit4]   2> 298659 T1299 C:control_collection c:collection1
> oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class
> org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy:
> maxMergeAtOnce=27, maxMergeAtOnceExplicit=50, maxMergedSegmentMB=52.0615234375,
> floorSegmentMB=0.486328125, forceMergeDeletesPctAllowed=10.112383767408401,
> segmentsPerTier=19.0, maxCFSSegmentSizeMB=8.796093022207999E12,
> noCFSRatio=1.0
>    [junit4]   2> 298659 T1299 C:control_collection c:collection1
> oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits:
> num=1
>    [junit4]   2>                commit{dir=MockDirectoryWrapper(
> RAMDirectory@6041646a lockFactory=org.apache.lucene.store.
> SingleInstanceLockFactory@65279c84),segFN=segments_1,generation=1}
>    [junit4]   2> 298659 T1299 C:control_collection c:collection1
> oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>    [junit4]   2> 298664 T1299 C:control_collection c:collection1 oasup.
> UpdateRequestProcessorChain.init creating updateRequestProcessorChain
> "nodistrib"
>    [junit4]   2> 298664 T1299 C:control_collection c:collection1 oasup.
> UpdateRequestProcessorChain.init creating updateRequestProcessorChain
> "dedupe"
>    [junit4]   2> 298664 T1299 C:control_collection c:collection1 oasup.
> UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory
> into updateRequestProcessorChain "dedupe"
>    [junit4]   2> 298664 T1299 C:control_collection c:collection1 oasup.
> UpdateRequestProcessorChain.init creating updateRequestProcessorChain
> "stored_sig"
>    [junit4]   2> 298665 T1299 C:control_collection c:collection1 oasup.
> UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory
> into updateRequestProcessorChain "stored_sig"
>    [junit4]   2> 298665 T1299 C:control_collection c:collection1 oasup.
> UpdateRequestProcessorChain.init creating updateRequestProcessorChain
> "distrib-dup-test-chain-explicit"
>    [junit4]   2> 298665 T1299 C:control_collection c:collection1 oasup.
> UpdateRequestProcessorChain.init creating updateRequestProcessorChain
> "distrib-dup-test-chain-implicit"
>    [junit4]   2> 298665 T1299 C:control_collection c:collection1 oasup.
> UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory
> into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>    [junit4]   2> 298666 T1299 C:control_collection c:collection1
> oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain
> defined as default, creating implicit default
>    [junit4]   2> 298685 T1299 C:control_collection c:collection1
> oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
>    [junit4]   2> 298687 T1299 C:control_collection c:collection1
> oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
>    [junit4]   2> 298689 T1299 C:control_collection c:collection1
> oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
>    [junit4]   2> 298690 T1299 C:control_collection c:collection1
> oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
>    [junit4]   2> 298694 T1299 C:control_collection c:collection1
> oasc.RequestHandlers.initHandlersFromConfig Registered paths:
> /schema,/update/json/docs,/admin/threads,/config,/
> replication,/admin/file,/admin/logging,/admin/plugins,/
> update/csv,standard,/admin/ping,/admin/system,/get,/
> admin/luke,/admin/segments,/admin/mbeans,/admin/
> properties,/update,/update/json
>    [junit4]   2> 298696 T1299 C:control_collection c:collection1
> oasc.SolrCore.initStatsCache Using default statsCache cache:
> org.apache.solr.search.stats.LocalStatsCache
>    [junit4]   2> 298697 T1299 C:control_collection c:collection1
> oasu.UpdateHandler.<init> Using UpdateLog implementation:
> org.apache.solr.update.UpdateLog
>    [junit4]   2> 298697 T1299 C:control_collection c:collection1
> oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH
> numRecordsToKeep=100 maxNumLogsToKeep=10
>    [junit4]   2> 298699 T1299 C:control_collection c:collection1
> oasu.CommitTracker.<init> Hard AutoCommit: disabled
>    [junit4]   2> 298699 T1299 C:control_collection c:collection1
> oasu.CommitTracker.<init> Soft AutoCommit: disabled
>    [junit4]   2> 298699 T1299 C:control_collection c:collection1
> oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class
> org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy:
> maxMergeAtOnce=30, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=39.86328125,
> floorSegmentMB=2.154296875, forceMergeDeletesPctAllowed=25.115751995354497,
> segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12,
> noCFSRatio=0.0
>    [junit4]   2> 298700 T1299 C:control_collection c:collection1
> oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
>    [junit4]   2>                commit{dir=MockDirectoryWrapper(
> RAMDirectory@6041646a lockFactory=org.apache.lucene.store.
> SingleInstanceLockFactory@65279c84),segFN=segments_1,generation=1}
>    [junit4]   2> 298700 T1299 C:control_collection c:collection1
> oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>    [junit4]   2> 298700 T1299 C:control_collection c:collection1
> oass.SolrIndexSearcher.<init> Opening Searcher@445242be[collection1] main
>    [junit4]   2> 298701 T1299 C:control_collection c:collection1
> oascc.ZkStateReader.readConfigName Load collection config
> from:/collections/control_collection
>    [junit4]   2> 298701 T1299 C:control_collection c:collection1
> oascc.ZkStateReader.readConfigName path=/collections/control_collection
> configName=conf1 specified config exists in ZooKeeper
>    [junit4]   2> 298701 T1299 C:control_collection c:collection1
> oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based
> storage for the RestManager with znodeBase: /configs/conf1
>    [junit4]   2> 298702 T1299 C:control_collection c:collection1
> oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured
> ZooKeeperStorageIO with znodeBase: /configs/conf1
>    [junit4]   2> 298702 T1299 C:control_collection c:collection1
> oasr.RestManager.init Initializing RestManager with initArgs: {}
>    [junit4]   2> 298702 T1299 C:control_collection c:collection1
> oasr.ManagedResourceStorage.load Reading _rest_managed.json using
> ZooKeeperStorageIO:path=/configs/conf1
>    [junit4]   2> 298703 T1299 C:control_collection c:collection1
> oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data
> found for znode /configs/conf1/_rest_managed.json
>    [junit4]   2> 298703 T1299 C:control_collection c:collection1
> oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json
> using ZooKeeperStorageIO:path=/configs/conf1
>    [junit4]   2> 298703 T1299 C:control_collection c:collection1
> oasr.RestManager.init Initializing 0 registered ManagedResources
>    [junit4]   2> 298703 T1299 C:control_collection c:collection1
> oash.ReplicationHandler.inform Commits will be reserved for  10000
>    [junit4]   2> 298704 T1299 C:control_collection c:collection1
> oasc.CoreContainer.registerCore registering core: collection1
>    [junit4]   2> 298704 T1300 C:control_collection c:collection1
> oasc.SolrCore.registerSearcher [collection1] Registered new searcher
> Searcher@445242be[collection1] main{ExitableDirectoryReader(
> UninvertingDirectoryReader())}
>    [junit4]   2> 298704 T1303 C:control_collection S:shard1
> oasc.ZkController.register Register replica - core:collection1 address:
> http://127.0.0.1:50635 collection:control_collection shard:shard1
>    [junit4]   2> 298705 T1269 oass.SolrDispatchFilter.init
> user.dir=/usr/home/jenkins/jenkins-slave/workspace/
> Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
>    [junit4]   2> 298706 T1269 oass.SolrDispatchFilter.init
> SolrDispatchFilter.init() done
>    [junit4]   2> 298706 T1303 C:control_collection S:shard1
> oascc.SolrZkClient.makePath makePath: /collections/control_
> collection/leader_elect/shard1/election
>    [junit4]   2> 298710 T1303 C:control_collection S:shard1 oasc.
> ShardLeaderElectionContext.runLeaderProcess Running the leader process
> for shard shard1
>    [junit4]   2> 298710 T1269 oascc.SolrZkClient.
> createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
>    [junit4]   2> 298711 T1295 oasc.DistributedQueue$LatchWatcher.process
> NodeChildrenChanged fired on path /overseer/queue state SyncConnected
>    [junit4]   2> 298711 T1303 C:control_collection S:shard1 oasc.
> ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found
> to continue.
>    [junit4]   2> 298711 T1269 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>    [junit4]   2> 298712 T1303 C:control_collection S:shard1 oasc.
> ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try
> and sync
>    [junit4]   2> ASYNC  NEW_CORE C279 name=collection1
> org.apache.solr.core.SolrCore@1745c55d url=http://127.0.0.1:50635/
> collection1 node=127.0.0.1:50635_ C279_STATE=coll:control_collection
> core:collection1 props:{state=down, base_url=http://127.0.0.1:50635,
> node_name=127.0.0.1:50635_, core=collection1}
>    [junit4]   2> 298712 T1303 C:control_collection S:shard1 C279 P50635
> oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:50635/
> collection1/
>    [junit4]   2> 298712 T1296 oasc.Overseer$ClusterStateUpdater.run
> processMessage: queueSize: 1, message = {
>    [junit4]   2>          "operation":"leader",
>    [junit4]   2>          "shard":"shard1",
>    [junit4]   2>          "collection":"control_collection"} current
> state version: 1
>    [junit4]   2> 298712 T1307 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@6ea67455
> name:ZooKeeperConnection Watcher:127.0.0.1:50623/solr got event
> WatchedEvent state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 298712 T1303 C:control_collection S:shard1 C279 P50635
> oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
>    [junit4]   2> 298713 T1269 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>    [junit4]   2> 298713 T1303 C:control_collection S:shard1 C279 P50635
> oasc.SyncStrategy.syncToMe http://127.0.0.1:50635/collection1/ has no
> replicas
>    [junit4]   2> 298713 T1303 C:control_collection S:shard1 oasc.
> ShardLeaderElectionContext.runLeaderProcess I am the new leader:
> http://127.0.0.1:50635/collection1/ shard1
>    [junit4]   2> 298714 T1269 oascc.SolrZkClient.createZkACLProvider
> Using default ZkACLProvider
>    [junit4]   2> 298714 T1295 oascc.ZkStateReader$2.process A cluster
> state change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 298714 T1303 C:control_collection S:shard1
> oascc.SolrZkClient.makePath makePath: /collections/control_
> collection/leaders/shard1
>    [junit4]   2> 298714 T1269 oascc.ZkStateReader.
> createClusterStateWatchersAndUpdate Updating cluster state from
> ZooKeeper...
>    [junit4]   2> 298717 T1269 oasc.ChaosMonkey.monkeyLog monkey: init -
> expire sessions:false cause connection loss:false
>    [junit4]   2> 298718 T1269 oasc.AbstractFullDistribZkTestBase.createJettys
> Creating collection1 with stateFormat=2
>    [junit4]   2> 298719 T1269 oascc.SolrZkClient.
> createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
>    [junit4]   2> 298719 T1295 oasc.DistributedQueue$LatchWatcher.process
> NodeChildrenChanged fired on path /overseer/queue state SyncConnected
>    [junit4]   2> 298719 T1269 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>    [junit4]   2> 298720 T1296 oasc.Overseer$ClusterStateUpdater.run
> processMessage: queueSize: 1, message = {
>    [junit4]   2>          "operation":"leader",
>    [junit4]   2>          "shard":"shard1",
>    [junit4]   2>          "collection":"control_collection",
>    [junit4]   2>          "base_url":"http://127.0.0.1:50635",
>    [junit4]   2>          "core":"collection1",
>    [junit4]   2>          "state":"active"} current state version: 2
>    [junit4]   2> 298720 T1310 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@5c566433
> name:ZooKeeperConnection Watcher:127.0.0.1:50623/solr got event
> WatchedEvent state:SyncConnected type:None path:null path:null type:None
>    [junit4]   2> 298721 T1269 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>    [junit4]   2> 298721 T1269 oascc.SolrZkClient.createZkACLProvider
> Using default ZkACLProvider
>    [junit4]   2> 298722 T1295 oasc.DistributedQueue$LatchWatcher.process
> NodeChildrenChanged fired on path /overseer/queue state SyncConnected
>    [junit4]   2> 298723 T1296 oasc.Overseer$ClusterStateUpdater.run
> processMessage: queueSize: 1, message = {
>    [junit4]   2>          "operation":"create",
>    [junit4]   2>          "name":"collection1",
>    [junit4]   2>          "numShards":"1",
>    [junit4]   2>          "stateFormat":"2"} current state version: 2
>    [junit4]   2> 298723 T1296 oasco.ClusterStateMutator.createCollection
> building a new cName: collection1
>    [junit4]   2> 298724 T1295 oascc.ZkStateReader$2.process A cluster
> state change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 298724 T1307 oascc.ZkStateReader$2.process A cluster
> state change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 298769 T1303 C:control_collection S:shard1
> oasc.ZkController.register We are http://127.0.0.1:50635/collection1/ and
> leader is http://127.0.0.1:50635/collection1/
>    [junit4]   2> 298770 T1303 C:control_collection S:shard1
> oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=
> http://127.0.0.1:50635
>    [junit4]   2> 298770 T1303 C:control_collection S:shard1
> oasc.ZkController.checkRecovery I am the leader, no recovery necessary
>    [junit4]   2> 298770 T1303 C:control_collection S:shard1 c:collection1
> oasc.ZkController.publish publishing core=collection1 state=active
> collection=control_collection
>    [junit4]   2> 298770 T1303 C:control_collection S:shard1 c:collection1
> oasc.ZkController.publish numShards not found on descriptor - reading it
> from system property
>    [junit4]   2> 298771 T1295 oasc.DistributedQueue$LatchWatcher.process
> NodeChildrenChanged fired on path /overseer/queue state SyncConnected
>    [junit4]   2> 298772 T1296 oasc.Overseer$ClusterStateUpdater.run
> processMessage: queueSize: 1, message = {
>    [junit4]   2>          "collection":"control_collection",
>    [junit4]   2>          "core_node_name":"core_node1",
>    [junit4]   2>          "state":"active",
>    [junit4]   2>          "shard":"shard1",
>    [junit4]   2>          "base_url":"http://127.0.0.1:50635",
>    [junit4]   2>          "roles":null,
>    [junit4]   2>          "node_name":"127.0.0.1:50635_",
>    [junit4]   2>          "core":"collection1",
>    [junit4]   2>          "operation":"state",
>    [junit4]   2>          "numShards":"1"} current state version: 3
>    [junit4]   2> 298773 T1296 oasco.ReplicaMutator.updateState Update
> state numShards=1 message={
>    [junit4]   2>          "collection":"control_collection",
>    [junit4]   2>          "core_node_name":"core_node1",
>    [junit4]   2>          "state":"active",
>    [junit4]   2>          "shard":"shard1",
>    [junit4]   2>          "base_url":"http://127.0.0.1:50635",
>    [junit4]   2>          "roles":null,
>    [junit4]   2>          "node_name":"127.0.0.1:50635_",
>    [junit4]   2>          "core":"collection1",
>    [junit4]   2>          "operation":"state",
>    [junit4]   2>          "numShards":"1"}
>    [junit4]   2> 298773 T1296 oasco.ZkStateWriter.writePendingUpdates
> going to create_collection /collections/collection1/state.json
>    [junit4]   2> 298774 T1307 oascc.ZkStateReader$2.process A cluster
> state change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 298774 T1295 oascc.ZkStateReader$2.process A cluster
> state change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 298878 T1307 oascc.ZkStateReader$2.process A cluster
> state change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 298878 T1295 oascc.ZkStateReader$2.process A cluster
> state change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
>    [junit4]   2> 300125 T1269 oas.SolrTestCaseJ4.writeCoreProperties
> Writing core.properties file to /usr/home/jenkins/jenkins-
> slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/
> solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/shard-1-001/cores/collection1
>    [junit4]   2> 300128 T1269 oasc.AbstractFullDistribZkTestBase.createJettys
> create jetty 1 in directory /usr/home/jenkins/jenkins-
> slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/
> solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/shard-1-001
>    [junit4]   2> 300128 T1269 oejs.Server.doStart jetty-8.1.10.v20130312
>    [junit4]   2> 300132 T1269 oejs.AbstractConnector.doStart Started
> SocketConnector@127.0.0.1:50644
>    [junit4]   2> 300132 T1269 oascse.JettySolrRunner$1.lifeCycleStarted
> Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/
> workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/
> test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/shard-1-001/cores, solr.data.dir=/usr/home/
> jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.
> x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/tempDir-001/jetty1, hostContext=/,
> solrconfig=solrconfig.xml, hostPort=50643}
>    [junit4]   2> 300133 T1269 oass.SolrDispatchFilter.init
> SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
>    [junit4]   2> 300133 T1269 oasc.SolrResourceLoader.<init> new
> SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-
> slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/
> solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/shard-1-001/'
>    [junit4]   2> 300171 T1269 oasc.SolrXmlConfig.fromFile Loading
> container configuration from /usr/home/jenkins/jenkins-
> slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/
> solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/shard-1-001/solr.xml
>    [junit4]   2> 300213 T1269 oasc.CorePropertiesLocator.<init>
> Config-defined core root directory: /usr/home/jenkins/jenkins-
> slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/
> solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/shard-1-001/cores
>    [junit4]   2> 300213 T1269 oasc.CoreContainer.<init> New CoreContainer
> 1370575573
>    [junit4]   2> 300214 T1269 oasc.CoreContainer.load Loading cores into
> CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/
> workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/
> test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/shard-1-001/]
>    [junit4]   2> 300214 T1269 oasc.CoreContainer.load loading shared
> library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-
> Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest
> FBFBECE5D5AABA29-001/shard-1-001/lib
>    [junit4]   2> 300215 T1269 oasc.SolrResourceLoader.addToClassLoader
> WARN Can

[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2900 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2900/

4 tests failed.
REGRESSION:  org.apache.solr.cloud.RecoveryZkTest.test

Error Message:
shard1 is not consistent.  Got 1023 from http://127.0.0.1:10050/p_/z/collection1lastClient and got 295 from http://127.0.0.1:10059/p_/z/collection1

Stack Trace:
java.lang.AssertionError: shard1 is not consistent.  Got 1023 from http://127.0.0.1:10050/p_/z/collection1lastClient and got 295 from http://127.0.0.1:10059/p_/z/collection1
	at __randomizedtesting.SeedInfo.seed([FBFBECE5D5AABA29:73AFD33F7B56D7D1]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.RecoveryZkTest.test(RecoveryZkTest.java:123)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:52540/fmq/gz/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:52540/fmq/gz/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([FBFBECE5D5AABA29:73AFD33F7B56D7D1]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:52540 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:50643/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:50643/collection1
	at __randomizedtesting.SeedInfo.seed([FBFBECE5D5AABA29:73AFD33F7B56D7D1]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:50643/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:50643 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:11223/repfacttest_c8n_1x3_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:11223/repfacttest_c8n_1x3_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([FBFBECE5D5AABA29:73AFD33F7B56D7D1]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:11223/repfacttest_c8n_1x3_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:11223 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9494 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/init-core-data-001
   [junit4]   2> 295434 T1269 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 295441 T1269 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 295441 T1270 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 295541 T1269 oasc.ZkTestServer.run start zk server on port:50623
   [junit4]   2> 295542 T1269 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 295542 T1269 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 295545 T1277 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6e5605f4 name:ZooKeeperConnection Watcher:127.0.0.1:50623 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 295545 T1269 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 295546 T1269 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 295546 T1269 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 295548 T1269 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 295549 T1269 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 295550 T1280 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@842caa6 name:ZooKeeperConnection Watcher:127.0.0.1:50623/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 295550 T1269 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 295551 T1269 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 295551 T1269 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 295553 T1269 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 295554 T1269 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 295555 T1269 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 295556 T1269 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 295557 T1269 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 295559 T1269 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 295560 T1269 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 295661 T1269 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 295662 T1269 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 295663 T1269 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 295663 T1269 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 295665 T1269 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 295665 T1269 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 295667 T1269 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 295667 T1269 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 295668 T1269 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 295669 T1269 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 295670 T1269 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 295670 T1269 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 295672 T1269 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 295672 T1269 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 295673 T1269 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 295674 T1269 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 295675 T1269 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 295675 T1269 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 296963 T1269 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores/collection1
   [junit4]   2> 296967 T1269 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 296971 T1269 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:50636
   [junit4]   2> 296971 T1269 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=50635, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/tempDir-001/control/data}
   [junit4]   2> 296971 T1269 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 296972 T1269 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/'
   [junit4]   2> 297015 T1269 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/solr.xml
   [junit4]   2> 297067 T1269 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores
   [junit4]   2> 297069 T1269 oasc.CoreContainer.<init> New CoreContainer 2110879613
   [junit4]   2> 297069 T1269 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/]
   [junit4]   2> 297069 T1269 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/lib
   [junit4]   2> 297070 T1269 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/lib).
   [junit4]   2> 297080 T1269 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 297080 T1269 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 297080 T1269 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 297080 T1269 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 297081 T1269 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 297081 T1269 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 297081 T1269 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 297081 T1269 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 297082 T1269 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 297082 T1269 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 297082 T1269 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 297089 T1269 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 297090 T1269 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 297090 T1269 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 297090 T1269 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 297091 T1269 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50623/solr
   [junit4]   2> 297091 T1269 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 297092 T1269 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 297092 T1269 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 297094 T1292 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@45cc06e8 name:ZooKeeperConnection Watcher:127.0.0.1:50623 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 297095 T1269 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 297096 T1269 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 297097 T1269 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 297098 T1295 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@34504d13 name:ZooKeeperConnection Watcher:127.0.0.1:50623/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 297098 T1269 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 297100 T1269 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 297101 T1269 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 297103 T1269 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 297104 T1269 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 297105 T1269 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 297107 T1269 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 297108 T1269 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 297109 T1269 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 297110 T1269 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50635_
   [junit4]   2> 297110 T1269 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50635_
   [junit4]   2> 297112 T1269 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 297113 T1269 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 297114 T1269 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 297115 T1269 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:50635_
   [junit4]   2> 297115 T1269 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 297116 T1269 oasc.Overseer.start Overseer (id=93607700461191171-127.0.0.1:50635_-n_0000000000) starting
   [junit4]   2> 297118 T1269 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 297122 T1269 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 297122 T1297 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 297122 T1269 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 297122 T1296 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 297126 T1269 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores
   [junit4]   2> 297128 T1269 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {shard=, collection=control_collection, coreNodeName=, dataDir=data/, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores/collection1/, config=solrconfig.xml, schema=schema.xml, loadOnStartup=true, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores/collection1, name=collection1}
   [junit4]   2> 297128 T1269 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores/collection1/
   [junit4]   2> 297128 T1269 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 297130 T1299 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 297130 T1299 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 297131 T1299 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 297131 T1295 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 297132 T1296 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50635",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:50635_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"} current state version: 0
   [junit4]   2> 297132 T1296 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50635",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:50635_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 297132 T1296 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 297133 T1296 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 297134 T1295 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 298131 T1299 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 298132 T1299 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 298132 T1299 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 298132 T1299 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 298133 T1299 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 298133 T1299 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores/collection1/'
   [junit4]   2> 298147 T1299 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 298149 T1299 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 298156 T1299 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 298172 T1299 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 298210 T1299 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 298311 T1299 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 298318 T1299 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 298623 T1299 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 298636 T1299 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 298638 T1299 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 298648 T1299 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 298651 T1299 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 298654 T1299 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 298655 T1299 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 298655 T1299 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 298655 T1299 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 298656 T1299 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 298656 T1299 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 298656 T1299 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 298656 T1299 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 298657 T1299 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores/collection1/data/
   [junit4]   2> 298657 T1299 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2ec2b6ef
   [junit4]   2> 298658 T1299 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores/collection1/data
   [junit4]   2> 298658 T1299 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores/collection1/data/index/
   [junit4]   2> 298658 T1299 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 298658 T1299 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/control-001/cores/collection1/data/index
   [junit4]   2> 298659 T1299 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=27, maxMergeAtOnceExplicit=50, maxMergedSegmentMB=52.0615234375, floorSegmentMB=0.486328125, forceMergeDeletesPctAllowed=10.112383767408401, segmentsPerTier=19.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 298659 T1299 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6041646a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@65279c84),segFN=segments_1,generation=1}
   [junit4]   2> 298659 T1299 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 298664 T1299 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 298664 T1299 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 298664 T1299 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 298664 T1299 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 298665 T1299 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 298665 T1299 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 298665 T1299 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 298665 T1299 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 298666 T1299 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 298685 T1299 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 298687 T1299 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 298689 T1299 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 298690 T1299 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 298694 T1299 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /schema,/update/json/docs,/admin/threads,/config,/replication,/admin/file,/admin/logging,/admin/plugins,/update/csv,standard,/admin/ping,/admin/system,/get,/admin/luke,/admin/segments,/admin/mbeans,/admin/properties,/update,/update/json
   [junit4]   2> 298696 T1299 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 298697 T1299 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 298697 T1299 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 298699 T1299 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 298699 T1299 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 298699 T1299 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=39.86328125, floorSegmentMB=2.154296875, forceMergeDeletesPctAllowed=25.115751995354497, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 298700 T1299 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6041646a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@65279c84),segFN=segments_1,generation=1}
   [junit4]   2> 298700 T1299 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 298700 T1299 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@445242be[collection1] main
   [junit4]   2> 298701 T1299 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 298701 T1299 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 298701 T1299 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 298702 T1299 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 298702 T1299 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 298702 T1299 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 298703 T1299 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 298703 T1299 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 298703 T1299 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 298703 T1299 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 298704 T1299 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 298704 T1300 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@445242be[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 298704 T1303 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:50635 collection:control_collection shard:shard1
   [junit4]   2> 298705 T1269 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 298706 T1269 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 298706 T1303 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 298710 T1303 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 298710 T1269 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 298711 T1295 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 298711 T1303 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 298711 T1269 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 298712 T1303 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C279 name=collection1 org.apache.solr.core.SolrCore@1745c55d url=http://127.0.0.1:50635/collection1 node=127.0.0.1:50635_ C279_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:50635, node_name=127.0.0.1:50635_, core=collection1}
   [junit4]   2> 298712 T1303 C:control_collection S:shard1 C279 P50635 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:50635/collection1/
   [junit4]   2> 298712 T1296 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 298712 T1307 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6ea67455 name:ZooKeeperConnection Watcher:127.0.0.1:50623/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 298712 T1303 C:control_collection S:shard1 C279 P50635 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 298713 T1269 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 298713 T1303 C:control_collection S:shard1 C279 P50635 oasc.SyncStrategy.syncToMe http://127.0.0.1:50635/collection1/ has no replicas
   [junit4]   2> 298713 T1303 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:50635/collection1/ shard1
   [junit4]   2> 298714 T1269 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 298714 T1295 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 298714 T1303 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 298714 T1269 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 298717 T1269 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 298718 T1269 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 298719 T1269 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 298719 T1295 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 298719 T1269 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 298720 T1296 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50635",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 298720 T1310 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5c566433 name:ZooKeeperConnection Watcher:127.0.0.1:50623/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 298721 T1269 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 298721 T1269 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 298722 T1295 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 298723 T1296 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 298723 T1296 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 298724 T1295 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 298724 T1307 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 298769 T1303 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:50635/collection1/ and leader is http://127.0.0.1:50635/collection1/
   [junit4]   2> 298770 T1303 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:50635
   [junit4]   2> 298770 T1303 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 298770 T1303 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 298770 T1303 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 298771 T1295 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 298772 T1296 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50635",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:50635_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"} current state version: 3
   [junit4]   2> 298773 T1296 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50635",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:50635_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 298773 T1296 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 298774 T1307 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 298774 T1295 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 298878 T1307 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 298878 T1295 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 300125 T1269 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores/collection1
   [junit4]   2> 300128 T1269 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001
   [junit4]   2> 300128 T1269 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 300132 T1269 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:50644
   [junit4]   2> 300132 T1269 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/tempDir-001/jetty1, hostContext=/, solrconfig=solrconfig.xml, hostPort=50643}
   [junit4]   2> 300133 T1269 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 300133 T1269 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/'
   [junit4]   2> 300171 T1269 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/solr.xml
   [junit4]   2> 300213 T1269 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores
   [junit4]   2> 300213 T1269 oasc.CoreContainer.<init> New CoreContainer 1370575573
   [junit4]   2> 300214 T1269 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/]
   [junit4]   2> 300214 T1269 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/lib
   [junit4]   2> 300215 T1269 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/lib).
   [junit4]   2> 300226 T1269 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 300226 T1269 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 300227 T1269 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 300227 T1269 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 300227 T1269 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 300227 T1269 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 300228 T1269 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 300228 T1269 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 300228 T1269 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 300228 T1269 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 300229 T1269 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 300234 T1269 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 300235 T1269 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 300235 T1269 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 300236 T1269 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 300236 T1269 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50623/solr
   [junit4]   2> 300236 T1269 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 300236 T1269 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 300237 T1269 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 300239 T1322 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@15d18950 name:ZooKeeperConnection Watcher:127.0.0.1:50623 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 300240 T1269 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 300242 T1269 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 300243 T1269 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 300245 T1325 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7a93df95 name:ZooKeeperConnection Watcher:127.0.0.1:50623/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 300245 T1269 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 300248 T1269 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 301258 T1269 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50643_
   [junit4]   2> 301259 T1269 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50643_
   [junit4]   2> 301262 T1269 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 301267 T1269 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores
   [junit4]   2> 301269 T1269 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores/collection1/, name=collection1, shard=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores/collection1, config=solrconfig.xml, loadOnStartup=true, transient=false, coreNodeName=, dataDir=data/, schema=schema.xml}
   [junit4]   2> 301270 T1269 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores/collection1/
   [junit4]   2> 301270 T1269 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 301273 T1326 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 301273 T1326 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 301275 T1295 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 301275 T1326 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 301275 T1326 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 301275 T1296 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50643",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:50643_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"} current state version: 5
   [junit4]   2> 301276 T1326 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 301276 T1296 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50643",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:50643_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 301276 T1296 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 301276 T1296 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 301277 T1326 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 301277 T1326 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 301278 T1296 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 301278 T1325 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 301279 T1325 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 302277 T1326 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 302278 T1326 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 302281 T1326 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 302282 T1326 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 302283 T1326 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 302284 T1326 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores/collection1/'
   [junit4]   2> 302320 T1326 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 302325 T1326 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 302356 T1326 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 302401 T1326 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 302535 T1326 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 302639 T1326 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 302662 T1326 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 303585 T1326 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 303613 T1326 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 303616 T1326 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 303637 T1326 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 303645 T1326 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 303652 T1326 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 303653 T1326 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 303654 T1326 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 303654 T1326 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 303655 T1326 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 303655 T1326 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 303655 T1326 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 303656 T1326 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 303656 T1326 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 303657 T1326 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2ec2b6ef
   [junit4]   2> 303658 T1326 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores/collection1/data
   [junit4]   2> 303658 T1326 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 303658 T1326 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 303659 T1326 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest FBFBECE5D5AABA29-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 303659 T1326 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=27, maxMergeAtOnceExplicit=50, maxMergedSegmentMB=52.0615234375, floorSegmentMB=0.486328125, forceMergeDeletesPctAllowed=10.112383767408401, segmentsPerTier=19.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 303661 T1326 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5be0b2e2 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7d00428d),segFN=segments_1,generation=1}
   [junit4]   2> 303661 T1326 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 303669 T1326 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 303669 T1326 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 303670 T1326 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 303670 T1326 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 303670 T1326 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 303671 T1326 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 303671 T1326 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 303671 T1326 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 303672 T1326 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 303704 T1326 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 303707 T1326 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 303710 T1326 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 303713 T1326 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 303719 T1326 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /schema,/update/json/docs,/admin/threads,/config,/replication,/admin/file,/admin/logging,/admin/plugins,/update/csv,standard,/admin/ping,/admin/system,/get,/admin/luke,/admin/segments,/admin/mbeans,/admin/properties,/update,/update/json
   [junit4]   2> 303722 T1326 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 303724 T1326 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 303724 T1326 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 303726 T1326 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 303726 T1326 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 303727 T1326 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=39.86328125, floorSegmentMB=2.154296875, forceMergeDeletesPctAllowed=25.115751995354497, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 303728 T1326 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5be0b2e2 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7d00428d),segFN=segments_1,generation=1}
   [junit4]   2> 303728 T1326 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 303729 T1326 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@83d513f[collection1] main
   [junit4]   2> 303729 T1326 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 303730 T1326 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 303730 T1326 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 303731 T1326 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 303731 T1326 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 303731 T1326 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 303732 T1326 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 303732 T1326 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 303732 T1326 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 303733 T1326 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 303734 T1327 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@83d513f[collection1] main{Exi

[...truncated too long message...]

96/
   [junit4]   2> 1572522 T4874 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:11212/
   [junit4]   2> 1572522 T4874 C:control_collection S:shard1 c:collection1 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 1572522 T4874 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:11208/
   [junit4]   2> 1572522 T4874 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:11223/
   [junit4]   2> 1572523 T4874 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:11196/
   [junit4]   2> 1572523 T4874 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:11212/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=FBFBECE5D5AABA29 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=es_ES -Dtests.timezone=America/Rankin_Inlet -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   73.2s J0 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:11223/repfacttest_c8n_1x3_shard1_replica2
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([FBFBECE5D5AABA29:73AFD33F7B56D7D1]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:11223/repfacttest_c8n_1x3_shard1_replica2
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 48 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:11223 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 53 more
   [junit4]   2> 1572544 T4874 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest FBFBECE5D5AABA29-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {range_facet_l=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), intDefault=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), text=PostingsFormat(name=Memory doPackFST= false), _version_=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), timestamp=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), multiDefault=FSTOrd50, a_t=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), id=Lucene50(blocksize=128)}, docValues:{timestamp=DocValuesFormat(name=Direct)}, sim=RandomSimilarityProvider(queryNorm=false,coord=crazy): {}, locale=es_ES, timezone=America/Rankin_Inlet
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=161230816,total=342884352
   [junit4]   2> NOTE: All tests run in this JVM: [TestIntervalFaceting, XmlUpdateRequestHandlerTest, CoreAdminRequestStatusTest, TestReplicaProperties, TestJettySolrRunner, TestRestManager, TestTrie, CollectionReloadTest, HardAutoCommitTest, MinimalSchemaTest, WordBreakSolrSpellCheckerTest, PolyFieldTest, TestHdfsUpdateLog, TestCursorMarkWithoutUniqueKey, QueryEqualityTest, OverseerRolesTest, TestQueryUtils, DirectUpdateHandlerOptimizeTest, TestFreeTextSuggestions, TriLevelCompositeIdRoutingTest, SpellPossibilityIteratorTest, TestAddFieldRealTimeGet, TestDFRSimilarityFactory, DeleteShardTest, PreAnalyzedUpdateProcessorTest, TestSchemaResource, HdfsSyncSliceTest, ZkCLITest, ShowFileRequestHandlerTest, TestQuerySenderListener, RequiredFieldsTest, SoftAutoCommitTest, TestIndexingPerformance, TestCloudPivotFacet, TestManagedSchemaFieldResource, TestSimpleTrackingShardHandler, TestLFUCache, CustomCollectionTest, TestInitQParser, DistributedQueryComponentCustomSortTest, MBeansHandlerTest, TimeZoneUtilsTest, SolrRequestParserTest, SolrTestCaseJ4Test, TestShardHandlerFactory, FileUtilsTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, ZkNodePropsTest, TestCSVResponseWriter, TestManagedStopFilterFactory, CacheHeaderTest, SolrCmdDistributorTest, TestConfig, TestBM25SimilarityFactory, TestManagedResource, MultiThreadedOCPTest, SyncSliceTest, TestCloudManagedSchemaConcurrent, ChaosMonkeyNothingIsSafeTest, ReplicationFactorTest]
   [junit4] Completed on J0 in 73.22s, 1 test, 1 error <<< FAILURES!

[...truncated 698 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 479 suites, 1903 tests, 3 errors, 1 failure, 61 ignored (21 assumptions)

Total time: 53 minutes 7 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 218990361 bytes
Compression is 0.0%
Took 40 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2899 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2899/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:37382/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37382/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([47F5277B5BCEDC6F:CFA118A1F532B197]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:37382 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22460/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22460/collection1
	at __randomizedtesting.SeedInfo.seed([47F5277B5BCEDC6F:CFA118A1F532B197]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22460/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:22460 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:16215/repfacttest_c8n_1x3_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:16215/repfacttest_c8n_1x3_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([47F5277B5BCEDC6F:CFA118A1F532B197]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:16215/repfacttest_c8n_1x3_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:16215 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9841 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/init-core-data-001
   [junit4]   2> 653060 T1976 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 653067 T1976 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 653068 T1977 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 653168 T1976 oasc.ZkTestServer.run start zk server on port:22447
   [junit4]   2> 653169 T1976 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 653170 T1976 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 653174 T1984 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@110ad99a name:ZooKeeperConnection Watcher:127.0.0.1:22447 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 653174 T1976 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 653174 T1976 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 653175 T1976 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 653177 T1976 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 653178 T1976 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 653179 T1987 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@30950d04 name:ZooKeeperConnection Watcher:127.0.0.1:22447/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 653179 T1976 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 653180 T1976 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 653180 T1976 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 653182 T1976 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 653183 T1976 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 653184 T1976 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 653186 T1976 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 653186 T1976 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 653189 T1976 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 653189 T1976 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 653291 T1976 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 653291 T1976 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 653293 T1976 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 653294 T1976 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 653295 T1976 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 653296 T1976 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 653298 T1976 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 653298 T1976 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 653300 T1976 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 653300 T1976 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 653302 T1976 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 653302 T1976 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 653304 T1976 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 653304 T1976 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 653306 T1976 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 653307 T1976 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 653308 T1976 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 653309 T1976 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 654775 T1976 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/control-001/cores/collection1
   [junit4]   2> 654779 T1976 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 654781 T1976 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:22455
   [junit4]   2> 654782 T1976 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=22454, hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/tempDir-001/control/data}
   [junit4]   2> 654783 T1976 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 654783 T1976 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/control-001/'
   [junit4]   2> 654821 T1976 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/control-001/solr.xml
   [junit4]   2> 654863 T1976 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/control-001/cores
   [junit4]   2> 654863 T1976 oasc.CoreContainer.<init> New CoreContainer 594929581
   [junit4]   2> 654863 T1976 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/control-001/]
   [junit4]   2> 654864 T1976 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/control-001/lib
   [junit4]   2> 654864 T1976 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/control-001/lib).
   [junit4]   2> 654874 T1976 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 654874 T1976 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 654875 T1976 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 654875 T1976 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 654875 T1976 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 654875 T1976 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 654876 T1976 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 654876 T1976 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 654876 T1976 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 654876 T1976 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 654876 T1976 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 654882 T1976 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 654883 T1976 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 654883 T1976 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 654883 T1976 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 654884 T1976 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:22447/solr
   [junit4]   2> 654884 T1976 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 654884 T1976 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 654885 T1976 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 654886 T1999 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@646e9be3 name:ZooKeeperConnection Watcher:127.0.0.1:22447 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 654887 T1976 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 654888 T1976 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 654889 T1976 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 654890 T2002 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1c3dae1f name:ZooKeeperConnection Watcher:127.0.0.1:22447/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 654890 T1976 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 654891 T1976 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 654893 T1976 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 654894 T1976 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 654895 T1976 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 654897 T1976 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 654898 T1976 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 654900 T1976 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 654901 T1976 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 654901 T1976 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:22454_
   [junit4]   2> 654902 T1976 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:22454_
   [junit4]   2> 654903 T1976 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 654904 T1976 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 654905 T1976 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 654906 T1976 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:22454_
   [junit4]   2> 654906 T1976 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 654908 T1976 oasc.Overseer.start Overseer (id=93606581219819523-127.0.0.1:22454_-n_0000000000) starting
   [junit4]   2> 654909 T1976 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 654914 T1976 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 654914 T2004 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 654914 T1976 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 654914 T2003 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 654917 T1976 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/control-001/cores
   [junit4]   2> 654919 T1976 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, coreNodeName=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/control-001/cores/collection1/, dataDir=data/, transient=false, collection=control_collection, loadOnStartup=true, config=solrconfig.xml, shard=, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/control-001/cores/collection1}
   [junit4]   2> 654919 T1976 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/control-001/cores/collection1/
   [junit4]   2> 654919 T1976 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 654920 T2006 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 654920 T2006 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 654921 T2006 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 654921 T2002 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 654922 T2003 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:22454_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22454",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down"} current state version: 0
   [junit4]   2> 654922 T2003 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:22454_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22454",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 654922 T2003 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 654923 T2003 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 654924 T2002 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 655921 T2006 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 655922 T2006 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 655922 T2006 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 655923 T2006 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 655923 T2006 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 655923 T2006 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/control-001/cores/collection1/'
   [junit4]   2> 655937 T2006 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 655939 T2006 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 655948 T2006 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 655966 T2006 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 656004 T2006 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 656105 T2006 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 656113 T2006 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 656566 T2006 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 656582 T2006 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 656585 T2006 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 656596 T2006 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 656600 T2006 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 656604 T2006 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 656605 T2006 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 656605 T2006 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 656606 T2006 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 656606 T2006 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 656606 T2006 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 656607 T2006 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 656607 T2006 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 656608 T2006 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/control-001/cores/collection1/data/
   [junit4]   2> 656608 T2006 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@405600d7
   [junit4]   2> 656609 T2006 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/control-001/cores/collection1/data
   [junit4]   2> 656609 T2006 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/control-001/cores/collection1/data/index/
   [junit4]   2> 656609 T2006 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 656611 T2006 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/control-001/cores/collection1/data/index
   [junit4]   2> 656611 T2006 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=50, maxMergeAtOnceExplicit=20, maxMergedSegmentMB=80.2119140625, floorSegmentMB=1.5, forceMergeDeletesPctAllowed=3.3180332404560553, segmentsPerTier=14.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1350957791224503
   [junit4]   2> 656612 T2006 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@62a9cba0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7977a7fa),segFN=segments_1,generation=1}
   [junit4]   2> 656612 T2006 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 656619 T2006 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 656619 T2006 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 656620 T2006 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 656620 T2006 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 656620 T2006 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 656620 T2006 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 656621 T2006 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 656621 T2006 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 656622 T2006 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 656644 T2006 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 656646 T2006 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 656649 T2006 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 656651 T2006 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 656657 T2006 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update/json/docs,/admin/luke,/config,standard,/admin/segments,/replication,/admin/system,/get,/admin/threads,/admin/plugins,/admin/file,/schema,/update,/admin/mbeans,/admin/logging,/update/json,/admin/ping,/admin/properties,/update/csv
   [junit4]   2> 656659 T2006 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 656661 T2006 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 656661 T2006 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 656663 T2006 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 656663 T2006 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 656664 T2006 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=14, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.33693801321823635]
   [junit4]   2> 656666 T2006 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@62a9cba0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7977a7fa),segFN=segments_1,generation=1}
   [junit4]   2> 656666 T2006 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 656667 T2006 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@3f3cc2fa[collection1] main
   [junit4]   2> 656667 T2006 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 656668 T2006 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 656668 T2006 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 656669 T2006 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 656669 T2006 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 656669 T2006 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 656670 T2006 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 656670 T2006 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 656671 T2006 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 656671 T2006 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 656672 T2007 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3f3cc2fa[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 656672 T2006 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 656673 T2010 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:22454 collection:control_collection shard:shard1
   [junit4]   2> 656674 T1976 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 656674 T1976 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 656674 T2010 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 656678 T1976 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 656678 T1976 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 656678 T2010 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 656680 T2014 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2a5a5614 name:ZooKeeperConnection Watcher:127.0.0.1:22447/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 656680 T1976 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 656680 T2002 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 656681 T1976 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 656681 T2010 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 656681 T1976 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 656681 T2010 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 656681 T2003 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C227 name=collection1 org.apache.solr.core.SolrCore@49048db8 url=http://127.0.0.1:22454/collection1 node=127.0.0.1:22454_ C227_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:22454_, base_url=http://127.0.0.1:22454, core=collection1, state=down}
   [junit4]   2> 656681 T2010 C:control_collection S:shard1 C227 P22454 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:22454/collection1/
   [junit4]   2> 656682 T2010 C:control_collection S:shard1 C227 P22454 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 656682 T2010 C:control_collection S:shard1 C227 P22454 oasc.SyncStrategy.syncToMe http://127.0.0.1:22454/collection1/ has no replicas
   [junit4]   2> 656682 T2010 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:22454/collection1/ shard1
   [junit4]   2> 656682 T2010 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 656683 T2002 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 656683 T2014 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 656684 T1976 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 656684 T1976 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 656685 T1976 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 656685 T1976 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 656687 T2017 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3ced76f4 name:ZooKeeperConnection Watcher:127.0.0.1:22447/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 656687 T1976 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 656687 T2002 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 656687 T1976 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 656688 T2003 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22454",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 656690 T2003 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 656691 T2003 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 656691 T2014 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 656691 T2002 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 656694 T2002 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 656695 T2003 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 656696 T2002 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 656696 T2014 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 656738 T2010 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:22454/collection1/ and leader is http://127.0.0.1:22454/collection1/
   [junit4]   2> 656739 T2010 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:22454
   [junit4]   2> 656739 T2010 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 656739 T2010 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 656739 T2010 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 656741 T2002 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 656742 T2003 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:22454_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22454",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active"} current state version: 4
   [junit4]   2> 656742 T2003 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:22454_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22454",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active"}
   [junit4]   2> 656846 T2002 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 656846 T2014 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 658577 T1976 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001/cores/collection1
   [junit4]   2> 658581 T1976 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001
   [junit4]   2> 658581 T1976 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 658584 T1976 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:22461
   [junit4]   2> 658584 T1976 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=22460, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001/cores}
   [junit4]   2> 658585 T1976 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 658585 T1976 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001/'
   [junit4]   2> 658629 T1976 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001/solr.xml
   [junit4]   2> 658684 T1976 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001/cores
   [junit4]   2> 658684 T1976 oasc.CoreContainer.<init> New CoreContainer 1807901597
   [junit4]   2> 658685 T1976 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001/]
   [junit4]   2> 658685 T1976 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001/lib
   [junit4]   2> 658686 T1976 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001/lib).
   [junit4]   2> 658697 T1976 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 658698 T1976 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 658698 T1976 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 658698 T1976 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 658698 T1976 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 658698 T1976 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 658699 T1976 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 658699 T1976 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 658699 T1976 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 658699 T1976 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 658700 T1976 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 658708 T1976 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 658708 T1976 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 658709 T1976 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 658709 T1976 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 658709 T1976 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:22447/solr
   [junit4]   2> 658709 T1976 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 658710 T1976 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 658710 T1976 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 658712 T2029 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@71de2711 name:ZooKeeperConnection Watcher:127.0.0.1:22447 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 658713 T1976 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 658714 T1976 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 658715 T1976 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 658717 T2032 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@520ccf14 name:ZooKeeperConnection Watcher:127.0.0.1:22447/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 658717 T1976 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 658720 T1976 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 659724 T1976 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:22460_
   [junit4]   2> 659725 T1976 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:22460_
   [junit4]   2> 659727 T1976 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 659730 T1976 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001/cores
   [junit4]   2> 659732 T1976 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {coreNodeName=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001/cores/collection1/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001/cores/collection1, collection=collection1, dataDir=data/, shard=, loadOnStartup=true, transient=false, config=solrconfig.xml, name=collection1, schema=schema.xml}
   [junit4]   2> 659733 T1976 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001/cores/collection1/
   [junit4]   2> 659733 T1976 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 659735 T2033 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 659735 T2033 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 659736 T2002 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 659737 T2033 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 659737 T2003 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:22460_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22460",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down"} current state version: 5
   [junit4]   2> 659737 T2033 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 659738 T2003 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:22460_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22460",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 659738 T2033 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 659738 T2003 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 659738 T2003 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 659739 T2033 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 659739 T2033 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 659739 T2003 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 659740 T2032 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 659740 T2032 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 660739 T2033 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 660740 T2033 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 660741 T2033 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 660741 T2033 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 660742 T2033 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 660742 T2033 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001/cores/collection1/'
   [junit4]   2> 660760 T2033 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 660763 T2033 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 660775 T2033 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 660799 T2033 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 660853 T2033 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 660954 T2033 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 660961 T2033 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 661256 T2033 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 661277 T2033 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 661280 T2033 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 661288 T2033 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 661292 T2033 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 661295 T2033 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 661296 T2033 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 661296 T2033 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 661297 T2033 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 661297 T2033 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 661297 T2033 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 661298 T2033 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 661298 T2033 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 661298 T2033 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 661298 T2033 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@405600d7
   [junit4]   2> 661299 T2033 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001/cores/collection1/data
   [junit4]   2> 661299 T2033 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 661300 T2033 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 661300 T2033 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 47F5277B5BCEDC6F-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 661300 T2033 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=50, maxMergeAtOnceExplicit=20, maxMergedSegmentMB=80.2119140625, floorSegmentMB=1.5, forceMergeDeletesPctAllowed=3.3180332404560553, segmentsPerTier=14.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1350957791224503
   [junit4]   2> 661301 T2033 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3d4a4473 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@56d3f974),segFN=segments_1,generation=1}
   [junit4]   2> 661301 T2033 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 661307 T2033 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 661307 T2033 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 661307 T2033 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 661308 T2033 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 661308 T2033 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 661308 T2033 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 661308 T2033 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 661309 T2033 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 661309 T2033 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 661330 T2033 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 661332 T2033 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 661334 T2033 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 661336 T2033 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 661342 T2033 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update/json/docs,/admin/luke,/config,standard,/admin/segments,/replication,/admin/system,/get,/admin/threads,/admin/plugins,/admin/file,/schema,/update,/admin/mbeans,/admin/logging,/update/json,/admin/ping,/admin/properties,/update/csv
   [junit4]   2> 661344 T2033 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 661345 T2033 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 661345 T2033 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 661347 T2033 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 661347 T2033 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 661348 T2033 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=14, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.33693801321823635]
   [junit4]   2> 661349 T2033 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3d4a4473 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@56d3f974),segFN=segments_1,generation=1}
   [junit4]   2> 661349 T2033 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 661349 T2033 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@52bd8264[collection1] main
   [junit4]   2> 661349 T2033 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 661350 T2033 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 661350 T2033 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 661350 T2033 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 661351 T2033 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 661351 T2033 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 661351 T2033 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 661351 T2033 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 661352 T2033 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 661352 T2033 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 661353 T2034 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@52bd8264[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 661353 T2033 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 661354 T2037 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:22460 collection:collection1 shard:shard1
   [junit4]   2> 661354 T1976 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 661354 T2037 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 661355 T1976 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 661358 T2037 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 661359 T2002 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 661359 T2037 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 661359 T2037 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C228 name=collection1 org.apache.solr.core.SolrCore@eba0df9 url=http://127.0.0.1:22460/collection1 node=127.0.0.1:22460_ C228_STATE=coll:collection1 core:collection1 props:{node_name=127.0.0.1:22460_, base_url=http://127.0.0.1:22460, core=collection1, state=down}
   [junit4]   2> 661360 T2037 C:collection1 S:shard1 C228 P22460 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:22460/collection1/
   [junit4]   2> 661360 T2003 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 661360 T2037 C:collection1 S:shard1 C228 P22460 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 661360 T2037 C:collection1 S:shard1 C228 P22460 oasc.SyncStrategy.syncToMe http://127.0.0.1:22460/collection1/ has no replicas
   [junit4]   2> 661360 T2037 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:22460/collection1/ shard1
   [junit4]   2> 661360 T2037 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 661361 T2003 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 661361 T2032 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 661362 T2032 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 661364 T2002 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 661364 T2003 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22460",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 661365 T2003 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 661366 T2032 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 661366 T2032 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 661414 T2037 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:22460/collection1/ and leader is http://127.0.0.1:22460/collection1/
   [junit4]   2> 661415 T2037 C:collection1 S:shard1 

[...truncated too long message...]

5 T6359 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 2 connections to: http://127.0.0.1:16257/
   [junit4]   2> 1519576 T6359 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:16228/
   [junit4]   2> 1519576 T6359 C:control_collection S:shard1 c:collection1 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 1519576 T6359 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16215/
   [junit4]   2> 1519577 T6359 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16264/
   [junit4]   2> 1519577 T6359 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16257/
   [junit4]   2> 1519577 T6359 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16228/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=47F5277B5BCEDC6F -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=is_IS -Dtests.timezone=America/Kralendijk -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   82.0s J3 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:16215/repfacttest_c8n_1x3_shard1_replica3
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([47F5277B5BCEDC6F:CFA118A1F532B197]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:16215/repfacttest_c8n_1x3_shard1_replica3
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 48 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:16215 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 53 more
   [junit4]   2> 1519603 T6359 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 47F5277B5BCEDC6F-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {timestamp=Lucene50(blocksize=128), intDefault=PostingsFormat(name=MockRandom), a_t=Lucene50(blocksize=128), range_facet_l=Lucene50(blocksize=128), id=PostingsFormat(name=LuceneVarGapFixedInterval), text=PostingsFormat(name=LuceneVarGapDocFreqInterval), _version_=PostingsFormat(name=MockRandom), multiDefault=Lucene50(blocksize=128)}, docValues:{timestamp=DocValuesFormat(name=Memory)}, sim=DefaultSimilarity, locale=is_IS, timezone=America/Kralendijk
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=172440416,total=427819008
   [junit4]   2> NOTE: All tests run in this JVM: [DocValuesMultiTest, ZkNodePropsTest, AddSchemaFieldsUpdateProcessorFactoryTest, ResponseLogComponentTest, BasicDistributedZk2Test, TestConfigSets, DistributedSuggestComponentTest, TestObjectReleaseTracker, HdfsSyncSliceTest, DirectSolrSpellCheckerTest, TestHdfsUpdateLog, TestFoldingMultitermQuery, TestRTGBase, OverseerTest, TimeZoneUtilsTest, DeleteInactiveReplicaTest, TestFuzzyAnalyzedSuggestions, TestHashQParserPlugin, TestReqParamsAPI, DocumentAnalysisRequestHandlerTest, TestHighlightDedupGrouping, TestInfoStreamLogging, ShardRoutingCustomTest, FieldMutatingUpdateProcessorTest, RegexBoostProcessorTest, StatsComponentTest, PreAnalyzedFieldTest, TestSurroundQueryParser, TestSolrQueryParser, TestOrdValues, RequestHandlersTest, DistribDocExpirationUpdateProcessorTest, TestExtendedDismaxParser, HdfsDirectoryFactoryTest, DeleteShardTest, DistributedQueryComponentCustomSortTest, RecoveryAfterSoftCommitTest, AnalysisAfterCoreReloadTest, RequiredFieldsTest, TestManagedStopFilterFactory, CSVRequestHandlerTest, RequestLoggingTest, TestInitParams, TestSweetSpotSimilarityFactory, LukeRequestHandlerTest, DistribJoinFromCollectionTest, TestRawResponseWriter, UUIDUpdateProcessorFallbackTest, TriLevelCompositeIdRoutingTest, TestIndexingPerformance, TestDynamicFieldResource, HardAutoCommitTest, TestSystemIdResolver, CloudExitableDirectoryReaderTest, TestSolrJ, SuggesterWFSTTest, TestStressReorder, CachingDirectoryFactoryTest, NumericFieldsTest, TestRemoteStreaming, TestCloudManagedSchemaConcurrent, TestReplicationHandler, CustomCollectionTest, ReplicationFactorTest]
   [junit4] Completed on J3 in 82.07s, 1 test, 1 error <<< FAILURES!

[...truncated 708 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 53 minutes 44 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 217369576 bytes
Compression is 0.0%
Took 43 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2898 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2898/

4 tests failed.
REGRESSION:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test

Error Message:
There were too many update fails (28 > 20) - we expect it can happen, but shouldn't easily

Stack Trace:
java.lang.AssertionError: There were too many update fails (28 > 20) - we expect it can happen, but shouldn't easily
	at __randomizedtesting.SeedInfo.seed([D2B365EA0131FD74:5AE75A30AFCD908C]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertFalse(Assert.java:68)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:230)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:34624/d_k/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:34624/d_k/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([D2B365EA0131FD74:5AE75A30AFCD908C]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:34624 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:49592/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:49592/collection1
	at __randomizedtesting.SeedInfo.seed([D2B365EA0131FD74:5AE75A30AFCD908C]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:49592/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:49592 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22741/av_/repfacttest_c8n_1x3_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22741/av_/repfacttest_c8n_1x3_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([D2B365EA0131FD74:5AE75A30AFCD908C]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22741/av_/repfacttest_c8n_1x3_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:22741 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9377 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/init-core-data-001
   [junit4]   2> 102389 T454 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /av_/
   [junit4]   2> 102395 T454 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 102396 T455 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 102496 T454 oasc.ZkTestServer.run start zk server on port:22737
   [junit4]   2> 102497 T454 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 102498 T454 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 102501 T462 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@94552fd name:ZooKeeperConnection Watcher:127.0.0.1:22737 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 102501 T454 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 102502 T454 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 102502 T454 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 102505 T454 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 102506 T454 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 102507 T465 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7930f0f9 name:ZooKeeperConnection Watcher:127.0.0.1:22737/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 102508 T454 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 102508 T454 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 102509 T454 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 102512 T454 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 102515 T454 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 102517 T454 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 102520 T454 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 102520 T454 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 102524 T454 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 102524 T454 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 102527 T454 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 102528 T454 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 102530 T454 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 102531 T454 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 102534 T454 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 102534 T454 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 102537 T454 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 102537 T454 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 102540 T454 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 102540 T454 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 102543 T454 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 102543 T454 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 102546 T454 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 102546 T454 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 102549 T454 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 102549 T454 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 102552 T454 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 102552 T454 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 103542 T454 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/control-001/cores/collection1
   [junit4]   2> 103545 T454 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 103548 T454 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:22742
   [junit4]   2> 103548 T454 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/av_, hostPort=22741, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/tempDir-001/control/data}
   [junit4]   2> 103549 T454 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 103549 T454 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/control-001/'
   [junit4]   2> 103580 T454 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/control-001/solr.xml
   [junit4]   2> 103617 T454 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/control-001/cores
   [junit4]   2> 103618 T454 oasc.CoreContainer.<init> New CoreContainer 386184238
   [junit4]   2> 103618 T454 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/control-001/]
   [junit4]   2> 103619 T454 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/control-001/lib
   [junit4]   2> 103619 T454 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/control-001/lib).
   [junit4]   2> 103629 T454 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 103629 T454 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 103630 T454 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 103630 T454 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 103630 T454 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 103630 T454 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 103631 T454 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 103631 T454 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 103631 T454 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 103632 T454 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 103632 T454 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 103637 T454 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 103638 T454 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 103638 T454 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 103638 T454 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 103639 T454 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:22737/solr
   [junit4]   2> 103639 T454 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 103639 T454 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 103640 T454 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 103642 T477 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3e3a021a name:ZooKeeperConnection Watcher:127.0.0.1:22737 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 103642 T454 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 103643 T454 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 103645 T454 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 103646 T480 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5c20d4db name:ZooKeeperConnection Watcher:127.0.0.1:22737/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 103647 T454 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 103648 T454 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 103652 T454 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 103655 T454 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 103658 T454 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 103661 T454 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 103664 T454 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 103667 T454 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 103668 T454 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 103670 T454 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:22741_av_
   [junit4]   2> 103671 T454 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:22741_av_
   [junit4]   2> 103674 T454 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 103676 T454 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 103678 T454 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 103680 T454 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:22741_av_
   [junit4]   2> 103680 T454 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 103683 T454 oasc.Overseer.start Overseer (id=93605607187087363-127.0.0.1:22741_av_-n_0000000000) starting
   [junit4]   2> 103686 T454 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 103695 T454 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 103696 T482 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 103696 T454 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 103697 T481 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 103703 T454 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/control-001/cores
   [junit4]   2> 103712 T454 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, loadOnStartup=true, collection=control_collection, schema=schema.xml, coreNodeName=, name=collection1, shard=, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/control-001/cores/collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/control-001/cores/collection1/, dataDir=data/}
   [junit4]   2> 103712 T454 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/control-001/cores/collection1/
   [junit4]   2> 103712 T454 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 103714 T484 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 103714 T484 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 103719 T480 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 103719 T484 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 103724 T481 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22741/av_",
   [junit4]   2> 	  "node_name":"127.0.0.1:22741_av_"} current state version: 0
   [junit4]   2> 103756 T481 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22741/av_",
   [junit4]   2> 	  "node_name":"127.0.0.1:22741_av_"}
   [junit4]   2> 103760 T481 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 103794 T481 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 103806 T480 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 104721 T484 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 104721 T484 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 104722 T484 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 104722 T484 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 104724 T484 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 104724 T484 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/control-001/cores/collection1/'
   [junit4]   2> 104758 T484 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 104775 T484 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 104799 T484 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 104823 T484 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 104862 T484 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 104864 T484 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 104886 T484 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 105413 T484 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 105453 T484 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 105457 T484 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 105509 T484 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 105518 T484 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 105526 T484 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 105528 T484 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 105529 T484 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 105530 T484 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 105532 T484 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 105533 T484 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 105534 T484 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 105534 T484 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 105535 T484 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/control-001/cores/collection1/data/
   [junit4]   2> 105536 T484 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2baec60c
   [junit4]   2> 105564 T484 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/control-001/cores/collection1/data
   [junit4]   2> 105565 T484 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/control-001/cores/collection1/data/index/
   [junit4]   2> 105565 T484 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 105566 T484 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/control-001/cores/collection1/data/index
   [junit4]   2> 105567 T484 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=23, maxMergeAtOnceExplicit=46, maxMergedSegmentMB=19.234375, floorSegmentMB=1.67578125, forceMergeDeletesPctAllowed=20.4478361701504, segmentsPerTier=45.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 105568 T484 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6ce08c10 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@a0ba648),segFN=segments_1,generation=1}
   [junit4]   2> 105568 T484 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 105581 T484 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 105582 T484 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 105583 T484 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 105583 T484 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 105584 T484 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 105584 T484 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 105585 T484 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 105586 T484 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 105587 T484 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 105627 T484 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 105643 T484 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 105649 T484 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 105654 T484 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 105672 T484 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/ping,/update/json/docs,/admin/mbeans,/admin/logging,/admin/luke,/update/csv,/admin/system,/get,/admin/file,/admin/segments,/admin/properties,/schema,/update,/config,standard,/admin/threads,/admin/plugins,/replication,/update/json
   [junit4]   2> 105675 T484 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 105677 T484 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 105678 T484 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 105679 T484 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 105680 T484 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 105682 T484 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1228413357, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 105683 T484 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6ce08c10 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@a0ba648),segFN=segments_1,generation=1}
   [junit4]   2> 105684 T484 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 105684 T484 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@5de98963[collection1] main
   [junit4]   2> 105685 T484 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 105687 T484 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 105687 T484 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 105699 T484 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 105699 T484 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 105700 T484 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 105701 T484 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 105701 T484 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 105702 T484 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 105702 T484 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 105705 T485 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5de98963[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 105710 T484 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 105711 T488 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:22741/av_ collection:control_collection shard:shard1
   [junit4]   2> 105712 T454 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 105712 T454 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 105731 T488 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 105733 T454 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 105734 T454 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 105736 T492 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@202e1df1 name:ZooKeeperConnection Watcher:127.0.0.1:22737/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 105736 T454 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 105737 T454 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 105738 T454 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 105739 T488 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 105741 T480 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 105742 T488 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 105743 T488 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C10 name=collection1 org.apache.solr.core.SolrCore@74b4dce5 url=http://127.0.0.1:22741/av_/collection1 node=127.0.0.1:22741_av_ C10_STATE=coll:control_collection core:collection1 props:{core=collection1, state=down, base_url=http://127.0.0.1:22741/av_, node_name=127.0.0.1:22741_av_}
   [junit4]   2> 105743 T488 C:control_collection S:shard1 C10 P22741 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:22741/av_/collection1/
   [junit4]   2> 105743 T481 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 105743 T488 C:control_collection S:shard1 C10 P22741 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 105744 T488 C:control_collection S:shard1 C10 P22741 oasc.SyncStrategy.syncToMe http://127.0.0.1:22741/av_/collection1/ has no replicas
   [junit4]   2> 105744 T488 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:22741/av_/collection1/ shard1
   [junit4]   2> 105745 T488 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 105747 T454 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 105754 T492 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 105754 T480 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 105760 T481 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22741/av_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 105763 T480 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 105765 T480 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 105765 T492 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 105801 T488 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:22741/av_/collection1/ and leader is http://127.0.0.1:22741/av_/collection1/
   [junit4]   2> 105802 T488 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:22741/av_
   [junit4]   2> 105802 T488 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 105802 T488 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 105803 T488 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 105804 T480 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 105806 T481 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22741/av_",
   [junit4]   2> 	  "node_name":"127.0.0.1:22741_av_"} current state version: 3
   [junit4]   2> 105807 T481 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22741/av_",
   [junit4]   2> 	  "node_name":"127.0.0.1:22741_av_"}
   [junit4]   2> 105912 T480 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 105912 T492 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 107259 T454 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001/cores/collection1
   [junit4]   2> 107261 T454 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001
   [junit4]   2> 107262 T454 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 107264 T454 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:22764
   [junit4]   2> 107265 T454 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=22763, solrconfig=solrconfig.xml, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/tempDir-001/jetty1, hostContext=/av_}
   [junit4]   2> 107265 T454 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 107266 T454 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001/'
   [junit4]   2> 107302 T454 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001/solr.xml
   [junit4]   2> 107343 T454 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001/cores
   [junit4]   2> 107344 T454 oasc.CoreContainer.<init> New CoreContainer 807977474
   [junit4]   2> 107344 T454 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001/]
   [junit4]   2> 107345 T454 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001/lib
   [junit4]   2> 107345 T454 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001/lib).
   [junit4]   2> 107355 T454 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 107355 T454 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 107356 T454 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 107356 T454 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 107356 T454 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 107357 T454 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 107357 T454 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 107357 T454 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 107358 T454 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 107358 T454 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 107358 T454 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 107363 T454 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 107364 T454 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 107365 T454 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 107365 T454 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 107365 T454 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:22737/solr
   [junit4]   2> 107366 T454 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 107366 T454 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 107367 T454 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 107369 T504 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5e7839da name:ZooKeeperConnection Watcher:127.0.0.1:22737 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 107370 T454 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 107370 T454 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 107373 T454 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 107374 T507 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@12dca9fc name:ZooKeeperConnection Watcher:127.0.0.1:22737/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 107375 T454 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 107382 T454 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 108391 T454 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:22763_av_
   [junit4]   2> 108392 T454 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:22763_av_
   [junit4]   2> 108397 T454 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 108400 T454 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001/cores
   [junit4]   2> 108402 T454 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {loadOnStartup=true, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001/cores/collection1/, coreNodeName=, collection=collection1, shard=, dataDir=data/, transient=false, config=solrconfig.xml, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001/cores/collection1, name=collection1}
   [junit4]   2> 108403 T454 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001/cores/collection1/
   [junit4]   2> 108403 T454 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 108404 T508 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 108405 T508 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 108406 T480 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 108406 T508 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 108408 T481 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22763/av_",
   [junit4]   2> 	  "node_name":"127.0.0.1:22763_av_"} current state version: 4
   [junit4]   2> 108409 T481 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22763/av_",
   [junit4]   2> 	  "node_name":"127.0.0.1:22763_av_"}
   [junit4]   2> 108409 T481 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 108409 T481 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard3
   [junit4]   2> 108411 T507 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 108411 T480 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 108411 T492 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 109407 T508 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 109408 T508 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 109409 T508 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 109410 T508 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 109411 T508 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 109411 T508 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001/cores/collection1/'
   [junit4]   2> 109431 T508 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 109434 T508 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 109454 T508 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 109482 T508 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 109544 T508 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 109546 T508 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 109578 T508 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 110017 T508 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 110036 T508 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 110038 T508 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 110060 T508 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 110065 T508 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 110070 T508 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 110071 T508 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 110071 T508 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 110072 T508 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 110073 T508 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 110073 T508 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 110073 T508 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 110074 T508 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 110074 T508 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 110075 T508 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2baec60c
   [junit4]   2> 110076 T508 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001/cores/collection1/data
   [junit4]   2> 110076 T508 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 110076 T508 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 110078 T508 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D2B365EA0131FD74-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 110079 T508 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=23, maxMergeAtOnceExplicit=46, maxMergedSegmentMB=19.234375, floorSegmentMB=1.67578125, forceMergeDeletesPctAllowed=20.4478361701504, segmentsPerTier=45.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 110079 T508 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@63620338 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@37c18481),segFN=segments_1,generation=1}
   [junit4]   2> 110080 T508 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 110134 T508 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 110135 T508 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 110136 T508 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 110136 T508 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 110136 T508 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 110137 T508 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 110137 T508 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 110137 T508 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 110138 T508 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 110151 T508 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 110155 T508 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 110158 T508 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 110160 T508 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 110188 T508 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/ping,/update/json/docs,/admin/mbeans,/admin/logging,/admin/luke,/update/csv,/admin/system,/get,/admin/file,/admin/segments,/admin/properties,/schema,/update,/config,standard,/admin/threads,/admin/plugins,/replication,/update/json
   [junit4]   2> 110190 T508 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 110191 T508 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 110191 T508 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 110193 T508 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 110193 T508 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 110195 T508 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1228413357, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 110196 T508 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@63620338 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@37c18481),segFN=segments_1,generation=1}
   [junit4]   2> 110196 T508 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 110197 T508 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@7f774441[collection1] main
   [junit4]   2> 110197 T508 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 110199 T508 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 110199 T508 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 110200 T508 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 110200 T508 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 110200 T508 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 110201 T508 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 110202 T508 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 110202 T508 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 110202 T508 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 110206 T509 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7f774441[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 110207 T508 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 110208 T512 C:collection1 S:shard3 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:22763/av_ collection:collection1 shard:shard3
   [junit4]   2> 110208 T454 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 110209 T454 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 110209 T512 C:collection1 S:shard3 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard3/election
   [junit4]   2> 110215 T512 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard3
   [junit4]   2> 110217 T480 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 110217 T512 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 110218 T512 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C11 name=collection1 org.apache.solr.core.SolrCore@49daad2b url=http://127.0.0.1:22763/av_/collection1 node=127.0.0.1:22763_av_ C11_STATE=coll:collection1 core:collection1 props:{core=collection1, state=down, base_url=http://127.0.0.1:22763/av_, node_name=127.0.0.1:22763_av_}
   [junit4]   2> 110218 T512 C:collection1 S:shard3 C11 P22763 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:22763/av_/collection1/
   [junit4]   2> 110218 T481 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 110218 T512 C:collection1 S:shard3 C11 P22763 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 110219 T512 C:collection1 S:shard3 C11 P22763 oasc.SyncStrategy.syncToMe http://127.0.0.1:22763/av_/collection1/ has no replicas
   [junit4]   2> 110219 T512 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:22763/av_/collection1/ shard3
   [junit4]   2> 110220 T512 C:collection1 S:shard3 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard3
   [junit4]   2> 110221 T507 oasc

[...truncated too long message...]

e/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1598541 T5214 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:59868/
   [junit4]   2> 1598541 T5214 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 6 connections to: http://127.0.0.1:49592/
   [junit4]   2> 1598542 T5214 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:59214/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=D2B365EA0131FD74 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=sv_SE -Dtests.timezone=America/Bogota -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   71.2s J3 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:49592/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([D2B365EA0131FD74:5AE75A30AFCD908C]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:49592/collection1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 49 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:49592 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 54 more
   [junit4]   2> 1598568 T5214 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest D2B365EA0131FD74-001
   [junit4]   2> 71227 T5213 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {id=PostingsFormat(name=MockRandom), a_t=PostingsFormat(name=LuceneVarGapFixedInterval), text=Lucene50(blocksize=128), timestamp=PostingsFormat(name=LuceneVarGapFixedInterval), range_facet_l=PostingsFormat(name=LuceneVarGapFixedInterval), multiDefault=PostingsFormat(name=Memory doPackFST= false), intDefault=PostingsFormat(name=LuceneVarGapFixedInterval), _version_=PostingsFormat(name=LuceneVarGapFixedInterval)}, docValues:{timestamp=DocValuesFormat(name=Memory)}, sim=DefaultSimilarity, locale=sv_SE, timezone=America/Bogota
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=170346632,total=387973120
   [junit4]   2> NOTE: All tests run in this JVM: [SignatureUpdateProcessorFactoryTest, UnloadDistributedZkTest, StressHdfsTest, ShowFileRequestHandlerTest, ResponseLogComponentTest, TestChildDocTransformer, TestSystemIdResolver, SpatialRPTFieldTypeTest, TestLRUStatsCache, IndexBasedSpellCheckerTest, BinaryUpdateRequestHandlerTest, SpatialHeatmapFacetsTest, TestZkChroot, SpellPossibilityIteratorTest, TestComplexPhraseQParserPlugin, DistributedQueryComponentOptimizationTest, ExternalFileFieldSortTest, TestShardHandlerFactory, TestRandomMergePolicy, AutoCommitTest, SpellCheckComponentTest, TestLMDirichletSimilarityFactory, TestCodecSupport, OverseerRolesTest, TestJmxMonitoredMap, TestInitParams, TestPhraseSuggestions, SegmentsInfoRequestHandlerTest, TestReload, TestFileDictionaryLookup, TestQueryUtils, TestReversedWildcardFilterFactory, CursorPagingTest, AssignTest, TestMissingGroups, TestFaceting, CoreAdminHandlerTest, TestTrie, HighlighterTest, TestFieldTypeCollectionResource, TestCSVResponseWriter, CurrencyFieldXmlFileTest, SSLMigrationTest, TestSimpleQParserPlugin, DefaultValueUpdateProcessorTest, TestObjectReleaseTracker, TestReplicaProperties, EnumFieldTest, TestSolrDynamicMBean, BasicDistributedZk2Test, TestRequestStatusCollectionAPI, TimeZoneUtilsTest, PolyFieldTest, TestManagedSynonymFilterFactory, DistributedSuggestComponentTest, DistributedQueueTest, SuggesterWFSTTest, TestComponentsName, TestCloudManagedSchemaConcurrent, TestDistributedSearch, CollectionsAPIDistributedZkTest, TestCloudManagedSchema, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J3 in 71.50s, 1 test, 1 error <<< FAILURES!

[...truncated 691 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 1 failure, 61 ignored (21 assumptions)

Total time: 55 minutes 6 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 219846757 bytes
Compression is 0.0%
Took 42 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2897 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2897/

5 tests failed.
REGRESSION:  org.apache.solr.handler.TestReqParamsAPI.test

Error Message:
Could not get expected value  'fomrequest' for path 'params/a' full output: null

Stack Trace:
java.lang.AssertionError: Could not get expected value  'fomrequest' for path 'params/a' full output: null
	at __randomizedtesting.SeedInfo.seed([4DEA682486127912:C5BE57FE28EE14EA]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.core.TestSolrConfigHandler.testForResponseElement(TestSolrConfigHandler.java:405)
	at org.apache.solr.handler.TestReqParamsAPI.testReqParams(TestReqParamsAPI.java:125)
	at org.apache.solr.handler.TestReqParamsAPI.test(TestReqParamsAPI.java:71)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.solr.handler.component.DistributedMLTComponentTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:60625/wip/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:60625/wip/collection1
	at __randomizedtesting.SeedInfo.seed([4DEA682486127912:C5BE57FE28EE14EA]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:566)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:943)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:558)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:606)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:588)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:567)
	at org.apache.solr.handler.component.DistributedMLTComponentTest.test(DistributedMLTComponentTest.java:126)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 51 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:19178/ju/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19178/ju/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([4DEA682486127912:C5BE57FE28EE14EA]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:19178 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:30763/_b/u/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:30763/_b/u/collection1
	at __randomizedtesting.SeedInfo.seed([4DEA682486127912:C5BE57FE28EE14EA]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:30763/_b/u/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:30763 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19236/zeii/repfacttest_c8n_1x3_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19236/zeii/repfacttest_c8n_1x3_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([4DEA682486127912:C5BE57FE28EE14EA]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19236/zeii/repfacttest_c8n_1x3_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:19236 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9364 lines...]
   [junit4] Suite: org.apache.solr.handler.TestReqParamsAPI
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/init-core-data-001
   [junit4]   2> 136455 T197 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 136455 T197 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /rbe/
   [junit4]   2> 136464 T197 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 136465 T198 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 136564 T197 oasc.ZkTestServer.run start zk server on port:31034
   [junit4]   2> 136565 T197 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 136567 T197 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 136570 T205 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3855f423 name:ZooKeeperConnection Watcher:127.0.0.1:31034 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 136571 T197 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 136572 T197 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 136572 T197 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 136575 T197 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 136576 T197 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 136581 T208 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@70dcbc89 name:ZooKeeperConnection Watcher:127.0.0.1:31034/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 136581 T197 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 136582 T197 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 136582 T197 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 136585 T197 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 136587 T197 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 136588 T197 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 136590 T197 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 136591 T197 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 136594 T197 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 136595 T197 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 136700 T197 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 136702 T197 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 136707 T197 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 136708 T197 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 136711 T197 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 136712 T197 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 136716 T197 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 136718 T197 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 136723 T197 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 136724 T197 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 136729 T197 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 136731 T197 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 136736 T197 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 136738 T197 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 136742 T197 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 136744 T197 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 136749 T197 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 136751 T197 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 140251 T197 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/control-001/cores/collection1
   [junit4]   2> 140262 T197 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 140270 T197 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:55955
   [junit4]   2> 140270 T197 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/control-001/cores, hostPort=55955, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/tempDir-001/control/data, hostContext=/rbe}
   [junit4]   2> 140271 T197 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@749cd006
   [junit4]   2> 140272 T197 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/control-001/'
   [junit4]   2> 140421 T197 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/control-001/solr.xml
   [junit4]   2> 140625 T197 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/control-001/cores
   [junit4]   2> 140626 T197 oasc.CoreContainer.<init> New CoreContainer 514480383
   [junit4]   2> 140626 T197 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/control-001/]
   [junit4]   2> 140627 T197 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/control-001/lib
   [junit4]   2> 140631 T197 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/control-001/lib).
   [junit4]   2> 140650 T197 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 140651 T197 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 140652 T197 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 140652 T197 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 140653 T197 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 140653 T197 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 140653 T197 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 140654 T197 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 140655 T197 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 140655 T197 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 140656 T197 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 140683 T197 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 140686 T197 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 140686 T197 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 140687 T197 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 140687 T197 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:31034/solr
   [junit4]   2> 140688 T197 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 140689 T197 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 140691 T197 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 140695 T220 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3e8aef88 name:ZooKeeperConnection Watcher:127.0.0.1:31034 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 140696 T197 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 140698 T197 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 140702 T197 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 140705 T223 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@582e177e name:ZooKeeperConnection Watcher:127.0.0.1:31034/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 140705 T197 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 140708 T197 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 140711 T197 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 140714 T197 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 140716 T197 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 140719 T197 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 140723 T197 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 140726 T197 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 140728 T197 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 140734 T197 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55955_rbe
   [junit4]   2> 140736 T197 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:55955_rbe
   [junit4]   2> 140738 T197 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 140741 T197 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 140744 T197 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 140748 T197 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:55955_rbe
   [junit4]   2> 140749 T197 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 140750 T197 oasc.Overseer.start Overseer (id=93604667402813443-127.0.0.1:55955_rbe-n_0000000000) starting
   [junit4]   2> 140754 T197 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 140773 T197 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 140774 T225 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 140774 T197 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 140775 T224 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 140784 T197 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/control-001/cores
   [junit4]   2> 140799 T197 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {shard=, name=collection1, loadOnStartup=true, config=solrconfig.xml, collection=control_collection, dataDir=data/, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/control-001/cores/collection1/, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/control-001/cores/collection1, transient=false, coreNodeName=}
   [junit4]   2> 140800 T197 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/control-001/cores/collection1/
   [junit4]   2> 140801 T197 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 140803 T227 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 140804 T227 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 140806 T223 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 140806 T227 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 140809 T224 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:55955_rbe",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:55955/rbe",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null} current state version: 0
   [junit4]   2> 140810 T224 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:55955_rbe",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:55955/rbe",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 140811 T224 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 140812 T224 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 140814 T223 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 141808 T227 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 141809 T227 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 141811 T227 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 141812 T227 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 141814 T227 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 141815 T227 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/control-001/cores/collection1/'
   [junit4]   2> 141846 T227 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 141853 T227 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 141872 T227 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 141937 T227 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 142040 T227 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 142142 T227 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 142215 T227 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 142962 T227 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 143009 T227 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 143015 T227 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 143057 T227 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 143071 T227 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 143082 T227 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 143085 T227 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 143086 T227 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 143086 T227 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 143088 T227 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 143089 T227 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 143089 T227 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 143090 T227 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 143091 T227 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/control-001/cores/collection1/data/
   [junit4]   2> 143091 T227 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@545319cc
   [junit4]   2> 143093 T227 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/control-001/cores/collection1/data
   [junit4]   2> 143093 T227 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/control-001/cores/collection1/data/index/
   [junit4]   2> 143094 T227 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 143094 T227 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/control-001/cores/collection1/data/index
   [junit4]   2> 143095 T227 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=49, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.20099856643417963]
   [junit4]   2> 143097 T227 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@750eb0fb lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@17856ac7),segFN=segments_1,generation=1}
   [junit4]   2> 143098 T227 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 143108 T227 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 143109 T227 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 143110 T227 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 143111 T227 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 143111 T227 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 143112 T227 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 143113 T227 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 143113 T227 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 143114 T227 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 143187 T227 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 143192 T227 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 143198 T227 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 143206 T227 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 143217 T227 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/ping,/admin/threads,/update/json,/admin/luke,/admin/file,/replication,/admin/properties,/update,/admin/mbeans,standard,/schema,/admin/logging,/admin/plugins,/admin/system,/get,/update/csv,/config,/update/json/docs,/admin/segments
   [junit4]   2> 143222 T227 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 143227 T227 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 143228 T227 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 143235 T227 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 143236 T227 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 143238 T227 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=38, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 143239 T227 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@750eb0fb lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@17856ac7),segFN=segments_1,generation=1}
   [junit4]   2> 143240 T227 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 143240 T227 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@1faeb8[collection1] main
   [junit4]   2> 143241 T227 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 143243 T227 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 143243 T227 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 143244 T227 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 143244 T227 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 143245 T227 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 143246 T227 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 143247 T227 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 143247 T227 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 143247 T227 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 143249 T228 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1faeb8[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 143249 T227 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 143250 T231 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:55955/rbe collection:control_collection shard:shard1
   [junit4]   2> 143251 T197 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 143251 T197 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 143253 T231 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 143255 T197 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 143256 T197 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 143258 T234 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@544187a0 name:ZooKeeperConnection Watcher:127.0.0.1:31034/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 143259 T197 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 143259 T231 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 143260 T197 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 143260 T197 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 143263 T223 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 143264 T231 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 143264 T231 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C13 name=collection1 org.apache.solr.core.SolrCore@11f56172 url=http://127.0.0.1:55955/rbe/collection1 node=127.0.0.1:55955_rbe C13_STATE=coll:control_collection core:collection1 props:{state=down, node_name=127.0.0.1:55955_rbe, core=collection1, base_url=http://127.0.0.1:55955/rbe}
   [junit4]   2> 143265 T231 C:control_collection S:shard1 C13 P55955 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:55955/rbe/collection1/
   [junit4]   2> 143265 T224 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 143266 T231 C:control_collection S:shard1 C13 P55955 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 143265 T197 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 143267 T231 C:control_collection S:shard1 C13 P55955 oasc.SyncStrategy.syncToMe http://127.0.0.1:55955/rbe/collection1/ has no replicas
   [junit4]   2> 143267 T197 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 143269 T231 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:55955/rbe/collection1/ shard1
   [junit4]   2> 143270 T223 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 143270 T234 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 143270 T197 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 143270 T231 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 143273 T197 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 143275 T237 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4b725081 name:ZooKeeperConnection Watcher:127.0.0.1:31034/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 143276 T197 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 143277 T197 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 143280 T223 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 143282 T224 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 143283 T224 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 143287 T224 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:55955/rbe",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 143289 T224 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 143291 T234 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 143291 T223 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 143297 T223 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 143298 T234 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 143298 T223 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 143335 T231 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:55955/rbe/collection1/ and leader is http://127.0.0.1:55955/rbe/collection1/
   [junit4]   2> 143335 T231 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:55955/rbe
   [junit4]   2> 143336 T231 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 143336 T231 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 143336 T231 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 143339 T223 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 143342 T224 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:55955_rbe",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:55955/rbe",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "roles":null} current state version: 4
   [junit4]   2> 143343 T224 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:55955_rbe",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:55955/rbe",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 143448 T223 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 143448 T234 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 146481 T197 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/shard-1-001/cores/collection1
   [junit4]   2> 146505 T197 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/shard-1-001
   [junit4]   2> 146506 T197 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 146513 T197 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:35051
   [junit4]   2> 146516 T197 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/rbe, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/shard-1-001/cores, hostPort=35051}
   [junit4]   2> 146518 T197 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@749cd006
   [junit4]   2> 146519 T197 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/shard-1-001/'
   [junit4]   2> 146671 T197 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/shard-1-001/solr.xml
   [junit4]   2> 146802 T197 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/shard-1-001/cores
   [junit4]   2> 146804 T197 oasc.CoreContainer.<init> New CoreContainer 1594767538
   [junit4]   2> 146804 T197 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/shard-1-001/]
   [junit4]   2> 146805 T197 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/shard-1-001/lib
   [junit4]   2> 146808 T197 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/shard-1-001/lib).
   [junit4]   2> 146834 T197 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 146835 T197 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 146836 T197 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 146836 T197 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 146837 T197 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 146838 T197 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 146839 T197 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 146839 T197 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 146840 T197 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 146840 T197 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 146841 T197 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 146861 T197 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 146863 T197 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 146864 T197 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 146864 T197 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 146865 T197 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:31034/solr
   [junit4]   2> 146865 T197 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 146866 T197 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 146867 T197 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 146869 T249 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@44d916e6 name:ZooKeeperConnection Watcher:127.0.0.1:31034 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 146870 T197 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 146871 T197 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 146875 T197 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 146877 T252 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@22fb957a name:ZooKeeperConnection Watcher:127.0.0.1:31034/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 146878 T197 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 146887 T197 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 147900 T197 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35051_rbe
   [junit4]   2> 147902 T197 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:35051_rbe
   [junit4]   2> 147909 T197 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 147915 T197 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/shard-1-001/cores
   [junit4]   2> 147926 T197 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, loadOnStartup=true, dataDir=data/, name=collection1, collection=collection1, coreNodeName=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/shard-1-001/cores/collection1, shard=, schema=schema.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/shard-1-001/cores/collection1/, transient=false}
   [junit4]   2> 147927 T197 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/shard-1-001/cores/collection1/
   [junit4]   2> 147928 T197 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 147935 T253 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 147936 T253 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 147939 T223 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 147942 T224 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:35051_rbe",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:35051/rbe",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "roles":null} current state version: 5
   [junit4]   2> 147943 T224 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:35051_rbe",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:35051/rbe",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 147943 T224 oasco.ReplicaMutator.updateState Collection already exists with numShards=2
   [junit4]   2> 147944 T224 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 147947 T224 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 147959 T253 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 147959 T253 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 147960 T253 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 147962 T253 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 147962 T253 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 147962 T253 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 147963 T253 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 147964 T253 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 147964 T253 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 147968 T253 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 147969 T253 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.TestReqParamsAPI 4DEA682486127912-001/shard-1-001/cores/collection1/'
   [junit4]   2> 148002 T253 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 148011 T253 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 148036 T253 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 148092 T253 C:collection1 c:collect

[...truncated too long message...]

ol_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 11 connections to: http://127.0.0.1:19236/zeii
   [junit4]   2> 1745441 T3579 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:19261/zeii
   [junit4]   2> 1745441 T3579 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 1 connections to: http://127.0.0.1:19215/zeii
   [junit4]   2> 1745441 T3579 C:control_collection S:shard1 c:collection1 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 1745442 T3579 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:19293/zeii
   [junit4]   2> 1745442 T3579 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:19236/zeii
   [junit4]   2> 1745442 T3579 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:19261/zeii
   [junit4]   2> 1745442 T3579 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:19215/zeii
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=4DEA682486127912 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=cs -Dtests.timezone=America/Kralendijk -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   79.8s J0 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19236/zeii/repfacttest_c8n_1x3_shard1_replica3
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([4DEA682486127912:C5BE57FE28EE14EA]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19236/zeii/repfacttest_c8n_1x3_shard1_replica3
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 48 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:19236 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 53 more
   [junit4]   2> 1745463 T3579 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4DEA682486127912-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {intDefault=Lucene50(blocksize=128), range_facet_l=Lucene50(blocksize=128), id=PostingsFormat(name=Memory doPackFST= false), text=FSTOrd50, a_t=Lucene50(blocksize=128), multiDefault=PostingsFormat(name=LuceneVarGapFixedInterval), timestamp=Lucene50(blocksize=128), _version_=Lucene50(blocksize=128)}, docValues:{timestamp=DocValuesFormat(name=Lucene50)}, sim=DefaultSimilarity, locale=cs, timezone=America/Kralendijk
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=185321288,total=279445504
   [junit4]   2> NOTE: All tests run in this JVM: [PreAnalyzedUpdateProcessorTest, TestBlobHandler, HdfsChaosMonkeySafeLeaderTest, TestManagedSchemaFieldTypeResource, TestReqParamsAPI, DistributedMLTComponentTest, SpatialFilterTest, InfoHandlerTest, DOMUtilTest, TestSortingResponseWriter, TestAnalyzeInfixSuggestions, HdfsRecoveryZkTest, TestConfigSets, TestOverriddenPrefixQueryForCustomFieldType, CoreAdminHandlerTest, TestSerializedLuceneMatchVersion, DistributedSpellCheckComponentTest, HighlighterMaxOffsetTest, DeleteLastCustomShardedReplicaTest, TestStressLucene, RegexBoostProcessorTest, HttpPartitionTest, TestPseudoReturnFields, BadIndexSchemaTest, SpellPossibilityIteratorTest, TestFastLRUCache, TestStressVersions, TestSort, TestLMJelinekMercerSimilarityFactory, SuggesterTSTTest, LeaderElectionTest, DistanceFunctionTest, DocValuesTest, MultiTermTest, DistribJoinFromCollectionTest, TestFieldCollectionResource, FileUtilsTest, OverseerStatusTest, TestLuceneMatchVersion, TestNamedUpdateProcessors, MultiThreadedOCPTest, AutoCommitTest, TestIntervalFaceting, DocumentAnalysisRequestHandlerTest, BasicZkTest, TestDocumentBuilder, TestManagedResource, CurrencyFieldOpenExchangeTest, TestGroupingSearch, DistributedTermsComponentTest, AnalysisErrorHandlingTest, TestReRankQParserPlugin, TestFuzzyAnalyzedSuggestions, TestSchemaSimilarityResource, BadCopyFieldTest, DefaultValueUpdateProcessorTest, TestIndexSearcher, DirectSolrSpellCheckerTest, ReplicationFactorTest]
   [junit4] Completed on J0 in 79.83s, 1 test, 1 error <<< FAILURES!

[...truncated 711 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 4 errors, 1 failure, 61 ignored (21 assumptions)

Total time: 58 minutes 54 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 222722426 bytes
Compression is 0.0%
Took 42 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2896 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2896/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:51146/ad_lr/q/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:51146/ad_lr/q/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([AB409680AE49383E:2314A95A00B555C6]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:51146 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28748/velr/s/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28748/velr/s/collection1
	at __randomizedtesting.SeedInfo.seed([AB409680AE49383E:2314A95A00B555C6]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28748/velr/s/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:28748 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22940/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22940/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([AB409680AE49383E:2314A95A00B555C6]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22940/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:22940 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9953 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/init-core-data-001
   [junit4]   2> 671811 T2102 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 671816 T2102 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 671816 T2103 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 671916 T2102 oasc.ZkTestServer.run start zk server on port:22936
   [junit4]   2> 671917 T2102 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 671917 T2102 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 671921 T2110 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@29d42291 name:ZooKeeperConnection Watcher:127.0.0.1:22936 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 671921 T2102 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 671922 T2102 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 671922 T2102 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 671924 T2102 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 671925 T2102 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 671926 T2113 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6487d71e name:ZooKeeperConnection Watcher:127.0.0.1:22936/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 671926 T2102 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 671927 T2102 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 671927 T2102 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 671928 T2102 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 671929 T2102 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 671930 T2102 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 671931 T2102 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 671932 T2102 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 671934 T2102 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 671934 T2102 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 672035 T2102 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 672036 T2102 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 672037 T2102 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 672037 T2102 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 672039 T2102 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 672039 T2102 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 672040 T2102 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 672041 T2102 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 672042 T2102 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 672042 T2102 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 672044 T2102 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 672044 T2102 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 672045 T2102 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 672046 T2102 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 672047 T2102 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 672048 T2102 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 672049 T2102 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 672049 T2102 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 673198 T2102 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/control-001/cores/collection1
   [junit4]   2> 673201 T2102 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 673204 T2102 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:22941
   [junit4]   2> 673204 T2102 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=22940, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/control-001/cores, hostContext=/}
   [junit4]   2> 673205 T2102 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 673205 T2102 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/control-001/'
   [junit4]   2> 673237 T2102 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/control-001/solr.xml
   [junit4]   2> 673286 T2102 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/control-001/cores
   [junit4]   2> 673286 T2102 oasc.CoreContainer.<init> New CoreContainer 1987784992
   [junit4]   2> 673286 T2102 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/control-001/]
   [junit4]   2> 673286 T2102 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/control-001/lib
   [junit4]   2> 673287 T2102 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/control-001/lib).
   [junit4]   2> 673297 T2102 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 673297 T2102 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 673298 T2102 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 673298 T2102 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 673298 T2102 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 673298 T2102 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 673298 T2102 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 673299 T2102 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 673299 T2102 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 673299 T2102 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 673299 T2102 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 673305 T2102 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 673305 T2102 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 673306 T2102 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 673306 T2102 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 673306 T2102 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:22936/solr
   [junit4]   2> 673306 T2102 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 673306 T2102 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 673307 T2102 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 673309 T2125 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@56c7eda0 name:ZooKeeperConnection Watcher:127.0.0.1:22936 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 673309 T2102 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 673310 T2102 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 673311 T2102 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 673322 T2128 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7f78b60b name:ZooKeeperConnection Watcher:127.0.0.1:22936/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 673322 T2102 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 673323 T2102 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 673325 T2102 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 673326 T2102 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 673328 T2102 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 673329 T2102 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 673331 T2102 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 673332 T2102 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 673333 T2102 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 673333 T2102 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:22940_
   [junit4]   2> 673334 T2102 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:22940_
   [junit4]   2> 673335 T2102 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 673336 T2102 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 673337 T2102 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 673338 T2102 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:22940_
   [junit4]   2> 673338 T2102 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 673339 T2102 oasc.Overseer.start Overseer (id=93603851893276675-127.0.0.1:22940_-n_0000000000) starting
   [junit4]   2> 673341 T2102 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 673345 T2102 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 673345 T2130 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 673345 T2102 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 673346 T2129 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 673349 T2102 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/control-001/cores
   [junit4]   2> 673351 T2102 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {shard=, schema=schema.xml, coreNodeName=, collection=control_collection, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/control-001/cores/collection1, loadOnStartup=true, dataDir=data/, config=solrconfig.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/control-001/cores/collection1/, transient=false, name=collection1}
   [junit4]   2> 673351 T2102 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/control-001/cores/collection1/
   [junit4]   2> 673351 T2102 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 673352 T2132 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 673352 T2132 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 673353 T2132 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 673353 T2128 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 673353 T2129 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22940",
   [junit4]   2> 	  "node_name":"127.0.0.1:22940_"} current state version: 0
   [junit4]   2> 673354 T2129 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22940",
   [junit4]   2> 	  "node_name":"127.0.0.1:22940_"}
   [junit4]   2> 673354 T2129 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 673354 T2129 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 673355 T2128 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 674353 T2132 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 674354 T2132 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 674354 T2132 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 674354 T2132 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 674355 T2132 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 674355 T2132 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/control-001/cores/collection1/'
   [junit4]   2> 674368 T2132 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 674370 T2132 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 674379 T2132 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 674396 T2132 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 674434 T2132 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 674535 T2132 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 674542 T2132 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 674843 T2132 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 674855 T2132 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 674857 T2132 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 674865 T2132 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 674868 T2132 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 674870 T2132 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 674871 T2132 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 674872 T2132 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 674872 T2132 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 674873 T2132 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 674873 T2132 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 674873 T2132 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 674873 T2132 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 674873 T2132 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/control-001/cores/collection1/data/
   [junit4]   2> 674874 T2132 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@41ee863
   [junit4]   2> 674874 T2132 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/control-001/cores/collection1/data
   [junit4]   2> 674874 T2132 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/control-001/cores/collection1/data/index/
   [junit4]   2> 674875 T2132 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 674875 T2132 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/control-001/cores/collection1/data/index
   [junit4]   2> 674875 T2132 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=43, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 674876 T2132 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@bf1054e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7c934a01),segFN=segments_1,generation=1}
   [junit4]   2> 674876 T2132 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 674880 T2132 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 674880 T2132 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 674880 T2132 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 674880 T2132 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 674881 T2132 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 674881 T2132 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 674881 T2132 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 674881 T2132 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 674881 T2132 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 674897 T2132 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 674898 T2132 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 674900 T2132 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 674902 T2132 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 674905 T2132 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/file,/admin/mbeans,/admin/logging,/admin/ping,/update/json/docs,/config,/get,/admin/threads,/admin/luke,/schema,/admin/properties,/replication,/admin/segments,/update,/admin/system,/update/csv,/admin/plugins,standard,/update/json
   [junit4]   2> 674907 T2132 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 674908 T2132 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 674908 T2132 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 674909 T2132 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 674909 T2132 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 674910 T2132 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=37, maxMergeAtOnceExplicit=50, maxMergedSegmentMB=8.111328125, floorSegmentMB=1.8037109375, forceMergeDeletesPctAllowed=16.785093067273394, segmentsPerTier=24.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 674910 T2132 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@bf1054e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7c934a01),segFN=segments_1,generation=1}
   [junit4]   2> 674911 T2132 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 674911 T2132 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@7a95226[collection1] main
   [junit4]   2> 674911 T2132 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 674912 T2132 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 674912 T2132 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 674912 T2132 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 674913 T2132 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 674913 T2132 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 674913 T2132 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 674913 T2132 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 674913 T2132 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 674914 T2132 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 674914 T2132 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 674914 T2133 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7a95226[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 674915 T2136 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:22940 collection:control_collection shard:shard1
   [junit4]   2> 674915 T2102 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 674915 T2102 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 674915 T2136 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 674918 T2136 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 674919 T2102 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 674919 T2102 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 674920 T2128 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 674920 T2136 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 674920 T2136 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 674921 T2129 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C437 name=collection1 org.apache.solr.core.SolrCore@5cc2c07f url=http://127.0.0.1:22940/collection1 node=127.0.0.1:22940_ C437_STATE=coll:control_collection core:collection1 props:{state=down, core=collection1, base_url=http://127.0.0.1:22940, node_name=127.0.0.1:22940_}
   [junit4]   2> 674921 T2136 C:control_collection S:shard1 C437 P22940 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:22940/collection1/
   [junit4]   2> 674920 T2140 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@16e87b6a name:ZooKeeperConnection Watcher:127.0.0.1:22936/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 674921 T2136 C:control_collection S:shard1 C437 P22940 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 674921 T2102 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 674921 T2136 C:control_collection S:shard1 C437 P22940 oasc.SyncStrategy.syncToMe http://127.0.0.1:22940/collection1/ has no replicas
   [junit4]   2> 674922 T2102 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 674922 T2128 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 674922 T2102 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 674922 T2136 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:22940/collection1/ shard1
   [junit4]   2> 674922 T2136 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 674924 T2102 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 674924 T2102 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 674925 T2102 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 674925 T2102 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 674925 T2128 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 674926 T2143 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@94b4162 name:ZooKeeperConnection Watcher:127.0.0.1:22936/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 674926 T2129 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22940",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 674926 T2102 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 674927 T2102 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 674928 T2128 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 674928 T2129 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 674929 T2129 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 674929 T2128 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 674929 T2140 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 674976 T2136 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:22940/collection1/ and leader is http://127.0.0.1:22940/collection1/
   [junit4]   2> 674977 T2136 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:22940
   [junit4]   2> 674977 T2136 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 674977 T2136 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 674977 T2136 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 674978 T2128 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 674979 T2129 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22940",
   [junit4]   2> 	  "node_name":"127.0.0.1:22940_"} current state version: 3
   [junit4]   2> 674979 T2129 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22940",
   [junit4]   2> 	  "node_name":"127.0.0.1:22940_"}
   [junit4]   2> 674980 T2129 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 674981 T2140 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 674981 T2128 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 675088 T2140 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 675088 T2128 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 676496 T2102 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001/cores/collection1
   [junit4]   2> 676500 T2102 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001
   [junit4]   2> 676501 T2102 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 676504 T2102 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:22948
   [junit4]   2> 676504 T2102 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostPort=22947, hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001/cores}
   [junit4]   2> 676505 T2102 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 676505 T2102 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001/'
   [junit4]   2> 676560 T2102 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001/solr.xml
   [junit4]   2> 676614 T2102 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001/cores
   [junit4]   2> 676614 T2102 oasc.CoreContainer.<init> New CoreContainer 898692719
   [junit4]   2> 676615 T2102 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001/]
   [junit4]   2> 676615 T2102 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001/lib
   [junit4]   2> 676616 T2102 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001/lib).
   [junit4]   2> 676626 T2102 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 676626 T2102 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 676626 T2102 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 676626 T2102 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 676626 T2102 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 676627 T2102 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 676627 T2102 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 676627 T2102 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 676627 T2102 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 676628 T2102 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 676628 T2102 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 676635 T2102 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 676636 T2102 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 676636 T2102 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 676636 T2102 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 676637 T2102 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:22936/solr
   [junit4]   2> 676637 T2102 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 676637 T2102 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 676638 T2102 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 676639 T2155 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6c1ffc19 name:ZooKeeperConnection Watcher:127.0.0.1:22936 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 676640 T2102 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 676640 T2102 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 676642 T2102 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 676643 T2158 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@132ee5ad name:ZooKeeperConnection Watcher:127.0.0.1:22936/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 676643 T2102 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 676645 T2102 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 677649 T2102 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:22947_
   [junit4]   2> 677650 T2102 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:22947_
   [junit4]   2> 677654 T2102 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 677656 T2102 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001/cores
   [junit4]   2> 677659 T2102 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, dataDir=data/, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001/cores/collection1, transient=false, schema=schema.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001/cores/collection1/, collection=collection1, coreNodeName=, shard=, loadOnStartup=true}
   [junit4]   2> 677659 T2102 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001/cores/collection1/
   [junit4]   2> 677659 T2102 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 677661 T2159 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 677661 T2159 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 677663 T2128 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 677663 T2159 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 677663 T2129 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22947",
   [junit4]   2> 	  "node_name":"127.0.0.1:22947_"} current state version: 5
   [junit4]   2> 677663 T2159 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 677664 T2129 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22947",
   [junit4]   2> 	  "node_name":"127.0.0.1:22947_"}
   [junit4]   2> 677664 T2129 oasco.ReplicaMutator.updateState Collection already exists with numShards=3
   [junit4]   2> 677664 T2159 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 677664 T2129 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard3
   [junit4]   2> 677665 T2159 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 677666 T2129 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 677666 T2159 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 677666 T2158 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 677667 T2158 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 678666 T2159 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 678667 T2159 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 678667 T2159 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 678667 T2159 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 678668 T2159 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 678668 T2159 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001/cores/collection1/'
   [junit4]   2> 678682 T2159 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 678684 T2159 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 678692 T2159 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 678708 T2159 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 678744 T2159 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 678845 T2159 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 678852 T2159 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 679130 T2159 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 679140 T2159 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 679142 T2159 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 679149 T2159 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 679160 T2159 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 679164 T2159 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 679164 T2159 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 679165 T2159 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 679165 T2159 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 679165 T2159 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 679165 T2159 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 679166 T2159 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 679166 T2159 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 679166 T2159 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 679166 T2159 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@41ee863
   [junit4]   2> 679167 T2159 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001/cores/collection1/data
   [junit4]   2> 679167 T2159 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 679167 T2159 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 679168 T2159 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest AB409680AE49383E-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 679168 T2159 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=43, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 679169 T2159 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@793b06c8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@123a72a6),segFN=segments_1,generation=1}
   [junit4]   2> 679169 T2159 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 679173 T2159 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 679174 T2159 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 679174 T2159 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 679174 T2159 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 679174 T2159 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 679174 T2159 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 679175 T2159 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 679175 T2159 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 679175 T2159 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 679187 T2159 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 679189 T2159 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 679190 T2159 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 679192 T2159 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 679197 T2159 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/file,/admin/mbeans,/admin/logging,/admin/ping,/update/json/docs,/config,/get,/admin/threads,/admin/luke,/schema,/admin/properties,/replication,/admin/segments,/update,/admin/system,/update/csv,/admin/plugins,standard,/update/json
   [junit4]   2> 679199 T2159 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 679200 T2159 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 679200 T2159 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 679201 T2159 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 679201 T2159 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 679202 T2159 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=37, maxMergeAtOnceExplicit=50, maxMergedSegmentMB=8.111328125, floorSegmentMB=1.8037109375, forceMergeDeletesPctAllowed=16.785093067273394, segmentsPerTier=24.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 679202 T2159 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@793b06c8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@123a72a6),segFN=segments_1,generation=1}
   [junit4]   2> 679203 T2159 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 679203 T2159 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@67910daa[collection1] main
   [junit4]   2> 679203 T2159 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 679204 T2159 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 679204 T2159 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 679204 T2159 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 679204 T2159 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 679205 T2159 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 679205 T2159 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 679205 T2159 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 679205 T2159 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 679206 T2159 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 679207 T2160 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@67910daa[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 679207 T2159 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 679208 T2163 C:collection1 S:shard3 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:22947 collection:collection1 shard:shard3
   [junit4]   2> 679208 T2102 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 679208 T2102 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 679209 T2163 C:collection1 S:shard3 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard3/election
   [junit4]   2> 679212 T2163 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard3
   [junit4]   2> 679214 T2128 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 679214 T2163 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 679214 T2163 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C438 name=collection1 org.apache.solr.core.SolrCore@1f2ff3a2 url=http://127.0.0.1:22947/collection1 node=127.0.0.1:22947_ C438_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, base_url=http://127.0.0.1:22947, node_name=127.0.0.1:22947_}
   [junit4]   2> 679214 T2163 C:collection1 S:shard3 C438 P22947 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:22947/collection1/
   [junit4]   2> 679214 T2129 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 679215 T2163 C:collection1 S:shard3 C438 P22947 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 679215 T2163 C:collection1 S:shard3 C438 P22947 oasc.SyncStrategy.syncToMe http://127.0.0.1:22947/collection1/ has no replicas
   [junit4]   2> 679215 T2163 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:22947/collection1/ shard3
   [junit4]   2> 679215 T2163 C:collection1 S:shard3 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard3
   [junit4]   2> 679216 T2129 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 679216 T2158 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 679217 T2158 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 679219 T2129 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22947",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 679220 T2129 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 679220 T2158 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 679221 T2158 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 679222 T2128 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 679269 T2163 C:collection1 S:shard3 oasc.ZkController.register We are http://127.0.0.1:22947/collection1/ and leader is http://127.0.0.1:22947/collection1/
   [junit4]   2> 679270 T2163 C:collection1 S:shard3 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:22947
   [junit4]   2> 679270 T2163 C:collection1 S:shard3 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 679270 T2163 C:collection1 S:shard3 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 679270 T2163 C:collection1 S:shard3 c:collection1 oasc.ZkController.publish numSh

[...truncated too long message...]

andler{/velr/s,null}
   [junit4]   2> 1510573 T5402 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:62281 62281
   [junit4]   2> 1510607 T5403 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:62281 62281
   [junit4]   2> 1510610 T5403 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 		2	/solr/collections/collection1/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1510610 T5402 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:34665/velr/s
   [junit4]   2> 1510611 T5402 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:28748/velr/s
   [junit4]   2> 1510611 T5402 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:21011/velr/s
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=AB409680AE49383E -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=no_NO -Dtests.timezone=Africa/Accra -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   67.8s J1 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28748/velr/s/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([AB409680AE49383E:2314A95A00B555C6]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28748/velr/s/collection1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 49 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:28748 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 54 more
   [junit4]   2> 1510639 T5402 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest AB409680AE49383E-001
   [junit4]   2> 67833 T5401 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50), sim=DefaultSimilarity, locale=no_NO, timezone=Africa/Accra
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=72927008,total=371195904
   [junit4]   2> NOTE: All tests run in this JVM: [PeerSyncTest, CloudMLTQParserTest, TestCollationField, OverseerRolesTest, TestSolrCoreProperties, AnalyticsQueryTest, TestFileDictionaryLookup, BlockDirectoryTest, SearchHandlerTest, ScriptEngineTest, TestRecovery, TestIBSimilarityFactory, TestIndexingPerformance, CursorMarkTest, SimpleMLTQParserTest, TestZkChroot, DefaultValueUpdateProcessorTest, TestComponentsName, SynonymTokenizerTest, OpenCloseCoreStressTest, DocumentAnalysisRequestHandlerTest, TestIntervalFaceting, TestCollectionAPI, CoreMergeIndexesAdminHandlerTest, CloudExitableDirectoryReaderTest, DocValuesTest, SharedFSAutoReplicaFailoverUtilsTest, TestPHPSerializedResponseWriter, TestRecoveryHdfs, AddBlockUpdateTest, DocumentBuilderTest, InfoHandlerTest, JsonLoaderTest, DistributedQueryComponentCustomSortTest, SliceStateTest, DistributedQueryElevationComponentTest, TestCSVLoader, ShardSplitTest, DistributedDebugComponentTest, TestManagedResourceStorage, SpellCheckComponentTest, CursorPagingTest, TestSearcherReuse, TestConfigSets, LeaderElectionTest, TestExceedMaxTermLength, TestShortCircuitedRequests, TestInfoStreamLogging, TestJettySolrRunner, TestSchemaManager, TestStandardQParsers, ResponseLogComponentTest, AnalysisAfterCoreReloadTest, TestAtomicUpdateErrorCases, ReplicationFactorTest, SpatialHeatmapFacetsTest, TestFieldTypeCollectionResource, TestStressUserVersions, TestCloudManagedSchemaConcurrent, TestDistributedSearch, MultiThreadedOCPTest, TestCloudManagedSchema, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J1 in 68.62s, 1 test, 1 error <<< FAILURES!

[...truncated 680 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 52 minutes 56 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 231796450 bytes
Compression is 0.0%
Took 44 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2895 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2895/

7 tests failed.
REGRESSION:  org.apache.solr.schema.TestCloudManagedSchemaConcurrent.test

Error Message:
QUERY FAILED: xpath=/response/lst[@name='responseHeader']/int[@name='status'][.='0']  request=/schema/copyfields/?wt=xml&updateTimeoutSecs=15  response=<?xml version="1.0" encoding="UTF-8"?> <response>  <lst name="responseHeader">   <int name="status">500</int>   <int name="QTime">46117</int> </lst> <lst name="error">   <str name="msg">7 out of 8 replicas failed to update their schema to version 25 within 15 seconds! Failed cores: [http://127.0.0.1:29575/ocp/s/collection1/, http://127.0.0.1:29583/ocp/s/collection1/, http://127.0.0.1:10348/ocp/s/collection1/, http://127.0.0.1:29603/ocp/s/collection1/, http://127.0.0.1:57708/ocp/s/collection1/, http://127.0.0.1:29609/ocp/s/collection1/, http://127.0.0.1:64149/ocp/s/collection1/]</str>   <str name="trace">org.apache.solr.common.SolrException: 7 out of 8 replicas failed to update their schema to version 25 within 15 seconds! Failed cores: [http://127.0.0.1:29575/ocp/s/collection1/, http://127.0.0.1:29583/ocp/s/collection1/, http://127.0.0.1:10348/ocp/s/collection1/, http://127.0.0.1:29603/ocp/s/collection1/, http://127.0.0.1:57708/ocp/s/collection1/, http://127.0.0.1:29609/ocp/s/collection1/, http://127.0.0.1:64149/ocp/s/collection1/]  at org.apache.solr.schema.ManagedIndexSchema.waitForSchemaZkVersionAgreement(ManagedIndexSchema.java:265)  at org.apache.solr.rest.schema.BaseFieldResource.waitForSchemaUpdateToPropagate(BaseFieldResource.java:120)  at org.apache.solr.rest.schema.CopyFieldCollectionResource.post(CopyFieldCollectionResource.java:189)  at org.restlet.resource.ServerResource.doHandle(ServerResource.java:437)  at org.restlet.resource.ServerResource.doConditionalHandle(ServerResource.java:350)  at org.restlet.resource.ServerResource.handle(ServerResource.java:952)  at org.restlet.resource.Finder.handle(Finder.java:246)  at org.restlet.routing.Filter.doHandle(Filter.java:159)  at org.restlet.routing.Filter.handle(Filter.java:206)  at org.restlet.routing.Router.doHandle(Router.java:431)  at org.restlet.routing.Router.handle(Router.java:648)  at org.restlet.routing.Filter.doHandle(Filter.java:159)  at org.restlet.routing.Filter.handle(Filter.java:206)  at org.restlet.routing.Filter.doHandle(Filter.java:159)  at org.restlet.routing.Filter.handle(Filter.java:206)  at org.restlet.routing.Filter.doHandle(Filter.java:159)  at org.restlet.engine.application.StatusFilter.doHandle(StatusFilter.java:155)  at org.restlet.routing.Filter.handle(Filter.java:206)  at org.restlet.routing.Filter.doHandle(Filter.java:159)  at org.restlet.routing.Filter.handle(Filter.java:206)  at org.restlet.engine.CompositeHelper.handle(CompositeHelper.java:211)  at org.restlet.engine.application.ApplicationHelper.handle(ApplicationHelper.java:84)  at org.restlet.Application.handle(Application.java:381)  at org.restlet.routing.Filter.doHandle(Filter.java:159)  at org.restlet.routing.Filter.handle(Filter.java:206)  at org.restlet.routing.Router.doHandle(Router.java:431)  at org.restlet.routing.Router.handle(Router.java:648)  at org.restlet.routing.Filter.doHandle(Filter.java:159)  at org.restlet.routing.Filter.handle(Filter.java:206)  at org.restlet.routing.Router.doHandle(Router.java:431)  at org.restlet.routing.Router.handle(Router.java:648)  at org.restlet.routing.Filter.doHandle(Filter.java:159)  at org.restlet.routing.Filter.handle(Filter.java:206)  at org.restlet.engine.CompositeHelper.handle(CompositeHelper.java:211)  at org.restlet.Component.handle(Component.java:392)  at org.restlet.Server.handle(Server.java:516)  at org.restlet.engine.ServerHelper.handle(ServerHelper.java:72)  at org.restlet.engine.adapter.HttpServerHelper.handle(HttpServerHelper.java:152)  at org.restlet.ext.servlet.ServerServlet.service(ServerServlet.java:1089)  at javax.servlet.http.HttpServlet.service(HttpServlet.java:848)  at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:669)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:457)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)  at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)  at org.eclipse.jetty.server.Dispatcher.forward(Dispatcher.java:276)  at org.eclipse.jetty.server.Dispatcher.forward(Dispatcher.java:103)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:102)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)  at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)  at org.eclipse.jetty.server.Server.handle(Server.java:368)  at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)  at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)  at org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:953)  at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1014)  at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:861)  at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240)  at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)  at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)  at java.lang.Thread.run(Thread.java:745) </str>   <int name="code">500</int> </lst> </response> 

Stack Trace:
java.lang.AssertionError: QUERY FAILED: xpath=/response/lst[@name='responseHeader']/int[@name='status'][.='0']  request=/schema/copyfields/?wt=xml&updateTimeoutSecs=15  response=<?xml version="1.0" encoding="UTF-8"?>
<response>

<lst name="responseHeader">
  <int name="status">500</int>
  <int name="QTime">46117</int>
</lst>
<lst name="error">
  <str name="msg">7 out of 8 replicas failed to update their schema to version 25 within 15 seconds! Failed cores: [http://127.0.0.1:29575/ocp/s/collection1/, http://127.0.0.1:29583/ocp/s/collection1/, http://127.0.0.1:10348/ocp/s/collection1/, http://127.0.0.1:29603/ocp/s/collection1/, http://127.0.0.1:57708/ocp/s/collection1/, http://127.0.0.1:29609/ocp/s/collection1/, http://127.0.0.1:64149/ocp/s/collection1/]</str>
  <str name="trace">org.apache.solr.common.SolrException: 7 out of 8 replicas failed to update their schema to version 25 within 15 seconds! Failed cores: [http://127.0.0.1:29575/ocp/s/collection1/, http://127.0.0.1:29583/ocp/s/collection1/, http://127.0.0.1:10348/ocp/s/collection1/, http://127.0.0.1:29603/ocp/s/collection1/, http://127.0.0.1:57708/ocp/s/collection1/, http://127.0.0.1:29609/ocp/s/collection1/, http://127.0.0.1:64149/ocp/s/collection1/]
	at org.apache.solr.schema.ManagedIndexSchema.waitForSchemaZkVersionAgreement(ManagedIndexSchema.java:265)
	at org.apache.solr.rest.schema.BaseFieldResource.waitForSchemaUpdateToPropagate(BaseFieldResource.java:120)
	at org.apache.solr.rest.schema.CopyFieldCollectionResource.post(CopyFieldCollectionResource.java:189)
	at org.restlet.resource.ServerResource.doHandle(ServerResource.java:437)
	at org.restlet.resource.ServerResource.doConditionalHandle(ServerResource.java:350)
	at org.restlet.resource.ServerResource.handle(ServerResource.java:952)
	at org.restlet.resource.Finder.handle(Finder.java:246)
	at org.restlet.routing.Filter.doHandle(Filter.java:159)
	at org.restlet.routing.Filter.handle(Filter.java:206)
	at org.restlet.routing.Router.doHandle(Router.java:431)
	at org.restlet.routing.Router.handle(Router.java:648)
	at org.restlet.routing.Filter.doHandle(Filter.java:159)
	at org.restlet.routing.Filter.handle(Filter.java:206)
	at org.restlet.routing.Filter.doHandle(Filter.java:159)
	at org.restlet.routing.Filter.handle(Filter.java:206)
	at org.restlet.routing.Filter.doHandle(Filter.java:159)
	at org.restlet.engine.application.StatusFilter.doHandle(StatusFilter.java:155)
	at org.restlet.routing.Filter.handle(Filter.java:206)
	at org.restlet.routing.Filter.doHandle(Filter.java:159)
	at org.restlet.routing.Filter.handle(Filter.java:206)
	at org.restlet.engine.CompositeHelper.handle(CompositeHelper.java:211)
	at org.restlet.engine.application.ApplicationHelper.handle(ApplicationHelper.java:84)
	at org.restlet.Application.handle(Application.java:381)
	at org.restlet.routing.Filter.doHandle(Filter.java:159)
	at org.restlet.routing.Filter.handle(Filter.java:206)
	at org.restlet.routing.Router.doHandle(Router.java:431)
	at org.restlet.routing.Router.handle(Router.java:648)
	at org.restlet.routing.Filter.doHandle(Filter.java:159)
	at org.restlet.routing.Filter.handle(Filter.java:206)
	at org.restlet.routing.Router.doHandle(Router.java:431)
	at org.restlet.routing.Router.handle(Router.java:648)
	at org.restlet.routing.Filter.doHandle(Filter.java:159)
	at org.restlet.routing.Filter.handle(Filter.java:206)
	at org.restlet.engine.CompositeHelper.handle(CompositeHelper.java:211)
	at org.restlet.Component.handle(Component.java:392)
	at org.restlet.Server.handle(Server.java:516)
	at org.restlet.engine.ServerHelper.handle(ServerHelper.java:72)
	at org.restlet.engine.adapter.HttpServerHelper.handle(HttpServerHelper.java:152)
	at org.restlet.ext.servlet.ServerServlet.service(ServerServlet.java:1089)
	at javax.servlet.http.HttpServlet.service(HttpServlet.java:848)
	at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:669)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:457)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
	at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
	at org.eclipse.jetty.server.Dispatcher.forward(Dispatcher.java:276)
	at org.eclipse.jetty.server.Dispatcher.forward(Dispatcher.java:103)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:102)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
	at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
	at org.eclipse.jetty.server.Server.handle(Server.java:368)
	at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
	at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
	at org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:953)
	at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1014)
	at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:861)
	at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240)
	at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
	at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
	at java.lang.Thread.run(Thread.java:745)
</str>
  <int name="code">500</int>
</lst>
</response>

	at __randomizedtesting.SeedInfo.seed([932E7C58AD80ADA2:1B7A4382037CC05A]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.schema.TestCloudManagedSchemaConcurrent.verifySuccess(TestCloudManagedSchemaConcurrent.java:107)
	at org.apache.solr.schema.TestCloudManagedSchemaConcurrent.copyField(TestCloudManagedSchemaConcurrent.java:153)
	at org.apache.solr.schema.TestCloudManagedSchemaConcurrent.access$500(TestCloudManagedSchemaConcurrent.java:45)
	at org.apache.solr.schema.TestCloudManagedSchemaConcurrent$Operation$5.execute(TestCloudManagedSchemaConcurrent.java:320)
	at org.apache.solr.schema.TestCloudManagedSchemaConcurrent.concurrentOperationsTest(TestCloudManagedSchemaConcurrent.java:450)
	at org.apache.solr.schema.TestCloudManagedSchemaConcurrent.test(TestCloudManagedSchemaConcurrent.java:252)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.update.SoftAutoCommitTest

Error Message:
Some resources were not closed, shutdown, or released.

Stack Trace:
java.lang.AssertionError: Some resources were not closed, shutdown, or released.
	at __randomizedtesting.SeedInfo.seed([932E7C58AD80ADA2]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.SolrTestCaseJ4.afterClass(SolrTestCaseJ4.java:234)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:799)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.solr.update.processor.TestNamedUpdateProcessors.test

Error Message:
Error from server at http://127.0.0.1:29369: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 404 Can not find: /admin/collections</title> </head> <body> <h2>HTTP ERROR: 404</h2> <p>Problem accessing /admin/collections. Reason: <pre>    Can not find: /admin/collections</pre></p> <hr /><i><small>Powered by Jetty://</small></i>                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:29369: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 404 Can not find: /admin/collections</title>
</head>
<body>
<h2>HTTP ERROR: 404</h2>
<p>Problem accessing /admin/collections. Reason:
<pre>    Can not find: /admin/collections</pre></p>
<hr /><i><small>Powered by Jetty://</small></i>
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([932E7C58AD80ADA2:1B7A4382037CC05A]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:525)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:152)
	at org.apache.solr.handler.TestBlobHandler.createSystemCollection(TestBlobHandler.java:112)
	at org.apache.solr.update.processor.TestNamedUpdateProcessors.test(TestNamedUpdateProcessors.java:86)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.update.processor.TestNamedUpdateProcessors

Error Message:
Some resources were not closed, shutdown, or released.

Stack Trace:
java.lang.AssertionError: Some resources were not closed, shutdown, or released.
	at __randomizedtesting.SeedInfo.seed([932E7C58AD80ADA2]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.SolrTestCaseJ4.afterClass(SolrTestCaseJ4.java:234)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:799)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:33528/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33528/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([932E7C58AD80ADA2:1B7A4382037CC05A]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:33528 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: 127.0.0.1:41489 failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:41489 failed to respond
	at __randomizedtesting.SeedInfo.seed([932E7C58AD80ADA2:1B7A4382037CC05A]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:41489 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: 127.0.0.1:24851 failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:24851 failed to respond
	at __randomizedtesting.SeedInfo.seed([932E7C58AD80ADA2:1B7A4382037CC05A]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:24851 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9365 lines...]
   [junit4] Suite: org.apache.solr.update.SoftAutoCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest 932E7C58AD80ADA2-001/init-core-data-001
   [junit4]   2> 116045 T295 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 116047 T295 oas.SolrTestCaseJ4.initCore ####initCore
   [junit4]   2> 116048 T295 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 116049 T295 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 116051 T295 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 116053 T295 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 116312 T295 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 116351 T295 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 116422 T295 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 116424 T295 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 116474 T295 oass.IndexSchema.readSchema [null] Schema name=test
   [junit4]   2> 117381 T295 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 117441 T295 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 117447 T295 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 117481 T295 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 117492 T295 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 117503 T295 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 117506 T295 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 117507 T295 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 117507 T295 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 117509 T295 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 117510 T295 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 117510 T295 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 117511 T295 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr
   [junit4]   2> 117511 T295 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/'
   [junit4]   2> 117536 T295 oasc.CoreContainer.<init> New CoreContainer 393660134
   [junit4]   2> 117536 T295 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/]
   [junit4]   2> 117537 T295 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/lib
   [junit4]   2> 117538 T295 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/lib).
   [junit4]   2> 117555 T295 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 600000
   [junit4]   2> 117556 T295 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 117556 T295 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 60000
   [junit4]   2> 117556 T295 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 117557 T295 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 117557 T295 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 117557 T295 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 117557 T295 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 117558 T295 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 117558 T295 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 117558 T295 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 117567 T295 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=30000&connTimeout=30000&retry=true
   [junit4]   2> 117568 T295 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 117568 T295 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 117568 T295 oasc.CoreContainer.load Node Name: testNode
   [junit4]   2> 117571 T295 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=collection1, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest 932E7C58AD80ADA2-001/init-core-data-001, transient=false, shard=shard1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, name=collection1, loadOnStartup=true, instanceDir=collection1, config=solrconfig.xml, schema=schema.xml}
   [junit4]   2> 117572 T296 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 117574 T296 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 117575 T296 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 117575 T296 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 117755 T296 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 117794 T296 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 117906 T296 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 117910 T296 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 117935 T296 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 118927 T296 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 118946 T296 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 118948 T296 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 118966 T296 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 118971 T296 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 118974 T296 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 118975 T296 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 118975 T296 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 118976 T296 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 118976 T296 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 118977 T296 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 118977 T296 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 118977 T296 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 118978 T296 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest 932E7C58AD80ADA2-001/init-core-data-001/
   [junit4]   2> 118978 T296 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@75749e15
   [junit4]   2> 118979 T296 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest 932E7C58AD80ADA2-001/init-core-data-001
   [junit4]   2> 118980 T296 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest 932E7C58AD80ADA2-001/init-core-data-001/index/
   [junit4]   2> 118980 T296 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest 932E7C58AD80ADA2-001/init-core-data-001/index' doesn't exist. Creating new index...
   [junit4]   2> 118981 T296 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest 932E7C58AD80ADA2-001/init-core-data-001/index
   [junit4]   2> 118981 T296 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=38, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.635920085406242]
   [junit4]   2> 118982 T296 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3808195b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1bb8c21a),segFN=segments_1,generation=1}
   [junit4]   2> 118983 T296 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 118992 T296 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 118993 T296 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 118993 T296 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 118994 T296 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 118994 T296 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 118994 T296 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 118995 T296 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 118995 T296 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 118995 T296 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 118996 T296 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 118997 T296 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 118997 T296 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 119025 T296 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 119028 T296 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 119031 T296 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 119034 T296 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 119044 T296 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 119050 T296 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: mltrh,spellCheckCompRH_Direct,standard,/admin/mbeans,/admin/system,/replication,lazy,/update/csv,/update/json,/admin/logging,/debug/dump,/search-facet-invariants,/admin/plugins,/get,/admin/ping,spellCheckWithWordbreak,/admin/segments,spellCheckCompRH1,/admin/file,/mlt,/admin/properties,spellCheckCompRH,/search-facet-def,/admin/threads,tvrh,dismax,/update,spellCheckWithWordbreak_Direct,/schema,/config,defaults,/update/json/docs,mock,/terms,/admin/luke
   [junit4]   2> 119052 T296 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 119055 T296 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 119059 T296 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 119060 T296 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 119061 T296 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 119061 T296 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 119062 T296 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@2d7292af
   [junit4]   2> 119063 T296 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3808195b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1bb8c21a),segFN=segments_1,generation=1}
   [junit4]   2> 119063 T296 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 119064 T296 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@443f7782[collection1] main
   [junit4]   2> 119065 T296 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 119067 T296 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 119067 T296 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 119067 T296 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 119131 T296 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 119251 T296 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 119253 T297 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 119255 T297 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 119255 T297 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 119255 T296 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 119255 T297 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 119258 T297 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 119260 T297 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 119262 T297 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 119262 T295 oas.SolrTestCaseJ4.initCore ####initCore end
   [junit4]   2> 119264 T297 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 119265 T297 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 119268 T297 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@443f7782[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 119270 T295 oas.SolrTestCaseJ4.setUp ###Starting testSoftAndHardCommitMaxTimeRapidAdds
   [junit4]   2> 119271 T295 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 119276 T295 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 119278 T295 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 119280 T295 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 119687 T295 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 119763 T295 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 119891 T295 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 119893 T295 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 119911 T295 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 165494 T295 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 165535 T295 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 165539 T295 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 165563 T295 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 165568 T295 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 165573 T295 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 165575 T295 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 165575 T295 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 165576 T295 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 165577 T295 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 165577 T295 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 165577 T295 oasc.CoreContainer.reload Reloading SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 165578 T295 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest 932E7C58AD80ADA2-001/init-core-data-001/
   [junit4]   2> 165579 T295 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@75749e15
   [junit4]   2> 165580 T295 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.update.SoftAutoCommitTest 932E7C58AD80ADA2-001/init-core-data-001/index/
   [junit4]   2> 165592 T295 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 165593 T295 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 165593 T295 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 165594 T295 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 165594 T295 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 165595 T295 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 165595 T295 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 165596 T295 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 165596 T295 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 165597 T295 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 165598 T295 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 165598 T295 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 165621 T295 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 165624 T295 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 165626 T295 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 165629 T295 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 165644 T295 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 165652 T295 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: mltrh,spellCheckCompRH_Direct,standard,/admin/mbeans,/admin/system,/replication,lazy,/update/csv,/update/json,/admin/logging,/debug/dump,/search-facet-invariants,/admin/plugins,/get,/admin/ping,spellCheckWithWordbreak,/admin/segments,spellCheckCompRH1,/admin/file,/mlt,/admin/properties,spellCheckCompRH,/search-facet-def,/admin/threads,tvrh,dismax,/update,spellCheckWithWordbreak_Direct,/schema,/config,defaults,/update/json/docs,mock,/terms,/admin/luke
   [junit4]   2> 165654 T295 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 165656 T295 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 165663 T295 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 165664 T295 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 165665 T295 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@a2b14bd[collection1] main
   [junit4]   2> 165666 T295 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 165666 T295 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 165667 T295 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 165667 T295 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 165670 T295 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 165682 T295 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 165683 T299 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 165684 T299 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 165684 T299 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 165685 T299 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 165685 T299 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 165686 T295 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
   [junit4]   2> 165686 T295 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
   [junit4]   2> 165686 T299 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 165687 T295 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter Closing old IndexWriter... core=collection1
   [junit4]   2> 165687 T299 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 165688 T295 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=10, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=18.26953125, floorSegmentMB=1.3505859375, forceMergeDeletesPctAllowed=24.319748176353148, segmentsPerTier=10.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7853453377607684
   [junit4]   2> 165688 T299 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 165689 T295 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3808195b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1bb8c21a),segFN=segments_1,generation=1}
   [junit4]   2> 165690 T299 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 165690 T295 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 165691 T295 c:collection1 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
   [junit4]   2> 165692 T299 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@a2b14bd[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 165693 T295 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@13b7dd80[collection1] main
   [junit4]   2> 165693 T295 c:collection1 oasc.CoreContainer.registerCore replacing core: collection1
   [junit4]   2> 165694 T295 c:collection1 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@6f2248c7
   [junit4]   2> 165694 T295 c:collection1 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=0,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=0,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=0,transaction_logs_total_number=0}
   [junit4]   2> 165695 T299 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@13b7dd80[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 165695 T295 c:collection1 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> ASYNC  NEW_CORE C26 name=collection1 org.apache.solr.core.SolrCore@69cea63e
   [junit4]   2> 165782 T295 c:collection1 C26 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[5000 (1497655808439091200)]} 0 68
   [junit4]   2> ASYNC  NEW_CORE C27 name=collection1 org.apache.solr.core.SolrCore@69cea63e
   [junit4]   2> 165795 T295 c:collection1 C27 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[5001 (1497655808522977280)]} 0 1
   [junit4]   2> 165799 T295 c:collection1 C27 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[5002 (1497655808527171584)]} 0 1
   [junit4]   2> 165802 T295 c:collection1 C27 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[5003 (1497655808530317312)]} 0 1
   [junit4]   2> 165806 T295 c:collection1 C27 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[5004 (1497655808534511616)]} 0 1
   [junit4]   2> 165808 T295 c:collection1 oas.SolrTestCaseJ4.tearDown ###Ending testSoftAndHardCommitMaxTimeRapidAdds
   [junit4]   2> 165815 T295 c:collection1 oas.SolrTestCaseJ4.setUp ###Starting testSoftAndHardCommitMaxTimeMixedAdds
   [junit4]   2> 165815 T295 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 165817 T295 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 165818 T295 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 165819 T295 c:collection1 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 165952 T295 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 165991 T295 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 166068 T295 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 166069 T295 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 166085 T295 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 166283 T301 c:collection1 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 166587 T301 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@35e0327e[collection1] main
   [junit4]   2> 166589 T301 c:collection1 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 166592 T299 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@35e0327e[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):C5)))}
   [junit4]   2> 166733 T295 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 166812 T295 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 166817 T295 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 166843 T295 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 166863 T295 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 166874 T295 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 166876 T295 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 166877 T295 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 166877 T295 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 166879 T295 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 166880 T295 c:col

[...truncated too long message...]

TestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 		2	/solr/collections/collection1/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1643454 T3785 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 6 connections to: http://127.0.0.1:41489/
   [junit4]   2> 1643454 T3785 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:41493/
   [junit4]   2> 1643454 T3785 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:41483/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=932E7C58AD80ADA2 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=in_ID -Dtests.timezone=America/Louisville -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   69.6s J0 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:41489 failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([932E7C58AD80ADA2:1B7A4382037CC05A]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:41489 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 1643476 T3785 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 932E7C58AD80ADA2-001
   [junit4]   2> 69616 T3784 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {timestamp=PostingsFormat(name=Memory doPackFST= true), multiDefault=PostingsFormat(name=Memory doPackFST= false), a_t=PostingsFormat(name=Memory doPackFST= true), id=Lucene50(blocksize=128), intDefault=PostingsFormat(name=Memory doPackFST= true), range_facet_l=PostingsFormat(name=Memory doPackFST= true), _version_=PostingsFormat(name=Memory doPackFST= true), text=PostingsFormat(name=LuceneVarGapFixedInterval)}, docValues:{timestamp=DocValuesFormat(name=Direct)}, sim=DefaultSimilarity, locale=in_ID, timezone=America/Louisville
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=185225304,total=304087040
   [junit4]   2> NOTE: All tests run in this JVM: [DateMathParserTest, OverriddenZkACLAndCredentialsProvidersTest, TestGroupingSearch, TestRandomMergePolicy, TestSort, TestDocBasedVersionConstraints, UniqFieldsUpdateProcessorFactoryTest, TestBadConfig, TestObjectReleaseTracker, AutoCommitTest, TestSearchPerf, PluginInfoTest, TestManagedResourceStorage, TestFastOutputStream, FieldMutatingUpdateProcessorTest, NumericFieldsTest, TestStandardQParsers, DistributedFacetPivotLongTailTest, IndexBasedSpellCheckerTest, TestSolrJ, DistributedQueryElevationComponentTest, CursorPagingTest, TestCursorMarkWithoutUniqueKey, TestFieldSortValues, ZkSolrClientTest, SpatialRPTFieldTypeTest, DOMUtilTest, TestStressUserVersions, SliceStateTest, LeaderElectionTest, DeleteLastCustomShardedReplicaTest, SuggestComponentTest, TestQueryTypes, TestFreeTextSuggestions, DirectUpdateHandlerTest, SpellCheckComponentTest, ExternalCollectionsTest, ChangedSchemaMergeTest, TestCharFilters, DeleteReplicaTest, SimpleMLTQParserTest, PolyFieldTest, TestPostingsSolrHighlighter, TestSolrXml, FacetPivotSmallTest, ResponseHeaderTest, TestFieldTypeCollectionResource, AnalysisErrorHandlingTest, TestSolrCoreProperties, TestImplicitCoreProperties, TestExactStatsCache, SimpleCollectionCreateDeleteTest, MultiThreadedOCPTest, TestFileDictionaryLookup, BasicFunctionalityTest, TestMaxScoreQueryParser, TestPivotHelperCode, ResponseLogComponentTest, HttpPartitionTest, TestReplicationHandler, UnloadDistributedZkTest, DistribDocExpirationUpdateProcessorTest, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J0 in 69.89s, 1 test, 1 error <<< FAILURES!

[...truncated 692 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 2 suite-level errors, 4 errors, 1 failure, 61 ignored (21 assumptions)

Total time: 55 minutes 17 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 217510061 bytes
Compression is 0.0%
Took 42 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2894 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2894/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:48409/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48409/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([A6DBB2D7042F4A95:2E8F8D0DAAD3276D]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:48409 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48539/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48539/collection1
	at __randomizedtesting.SeedInfo.seed([A6DBB2D7042F4A95:2E8F8D0DAAD3276D]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48539/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:48539 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:50207/zt_/rp/repfacttest_c8n_1x3_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:50207/zt_/rp/repfacttest_c8n_1x3_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([A6DBB2D7042F4A95:2E8F8D0DAAD3276D]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:50207/zt_/rp/repfacttest_c8n_1x3_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:50207 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9964 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/init-core-data-001
   [junit4]   2> 1132617 T2416 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /zt_/rp
   [junit4]   2> 1132621 T2416 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1132622 T2417 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1132722 T2416 oasc.ZkTestServer.run start zk server on port:50202
   [junit4]   2> 1132723 T2416 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1132724 T2416 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1132727 T2424 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4ffe3d82 name:ZooKeeperConnection Watcher:127.0.0.1:50202 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1132727 T2416 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1132728 T2416 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1132728 T2416 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1132730 T2416 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1132732 T2416 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1132733 T2427 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@530007a8 name:ZooKeeperConnection Watcher:127.0.0.1:50202/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1132734 T2416 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1132734 T2416 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1132734 T2416 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1132737 T2416 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1132738 T2416 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1132740 T2416 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1132742 T2416 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1132743 T2416 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1132745 T2416 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1132746 T2416 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1132848 T2416 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1132849 T2416 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1132850 T2416 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1132851 T2416 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1132852 T2416 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1132853 T2416 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1132854 T2416 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1132855 T2416 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1132856 T2416 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1132857 T2416 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1132858 T2416 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1132859 T2416 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1132860 T2416 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1132861 T2416 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1132862 T2416 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1132863 T2416 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1132864 T2416 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1132865 T2416 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1134404 T2416 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/control-001/cores/collection1
   [junit4]   2> 1134406 T2416 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1134410 T2416 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:50208
   [junit4]   2> 1134410 T2416 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/zt_/rp, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/tempDir-001/control/data, hostPort=50207}
   [junit4]   2> 1134411 T2416 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 1134411 T2416 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/control-001/'
   [junit4]   2> 1134444 T2416 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/control-001/solr.xml
   [junit4]   2> 1134486 T2416 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/control-001/cores
   [junit4]   2> 1134486 T2416 oasc.CoreContainer.<init> New CoreContainer 1897583399
   [junit4]   2> 1134486 T2416 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/control-001/]
   [junit4]   2> 1134486 T2416 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/control-001/lib
   [junit4]   2> 1134487 T2416 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/control-001/lib).
   [junit4]   2> 1134497 T2416 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1134497 T2416 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1134497 T2416 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1134497 T2416 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1134497 T2416 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1134498 T2416 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1134498 T2416 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1134498 T2416 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1134498 T2416 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1134499 T2416 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1134499 T2416 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1134504 T2416 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1134505 T2416 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1134505 T2416 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1134505 T2416 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1134506 T2416 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50202/solr
   [junit4]   2> 1134506 T2416 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1134506 T2416 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1134507 T2416 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1134508 T2439 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@39533a61 name:ZooKeeperConnection Watcher:127.0.0.1:50202 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1134509 T2416 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1134510 T2416 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1134511 T2416 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1134512 T2442 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5211b59d name:ZooKeeperConnection Watcher:127.0.0.1:50202/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1134513 T2416 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1134514 T2416 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1134516 T2416 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1134517 T2416 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1134519 T2416 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1134520 T2416 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1134522 T2416 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1134523 T2416 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1134524 T2416 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1134525 T2416 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50207_zt_%2Frp
   [junit4]   2> 1134525 T2416 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50207_zt_%2Frp
   [junit4]   2> 1134526 T2416 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1134527 T2416 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1134528 T2416 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1134529 T2416 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:50207_zt_%2Frp
   [junit4]   2> 1134529 T2416 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1134530 T2416 oasc.Overseer.start Overseer (id=93602473946251267-127.0.0.1:50207_zt_%2Frp-n_0000000000) starting
   [junit4]   2> 1134532 T2416 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1134536 T2416 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 1134536 T2444 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1134536 T2416 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1134536 T2443 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1134539 T2416 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/control-001/cores
   [junit4]   2> 1134542 T2416 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {dataDir=data/, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/control-001/cores/collection1/, coreNodeName=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/control-001/cores/collection1, shard=, name=collection1, config=solrconfig.xml, transient=false, loadOnStartup=true, collection=control_collection, schema=schema.xml}
   [junit4]   2> 1134542 T2416 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/control-001/cores/collection1/
   [junit4]   2> 1134542 T2416 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1134543 T2446 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1134543 T2446 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1134544 T2446 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1134544 T2442 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1134544 T2443 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50207/zt_/rp",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:50207_zt_%2Frp",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null} current state version: 0
   [junit4]   2> 1134545 T2443 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50207/zt_/rp",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:50207_zt_%2Frp",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 1134545 T2443 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 1134545 T2443 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1134546 T2442 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1135545 T2446 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1135545 T2446 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1135546 T2446 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1135546 T2446 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1135547 T2446 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1135547 T2446 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/control-001/cores/collection1/'
   [junit4]   2> 1135567 T2446 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1135569 T2446 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1135580 T2446 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1135603 T2446 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1135666 T2446 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1135768 T2446 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1135778 T2446 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1136280 T2446 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1136299 T2446 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1136302 T2446 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1136313 T2446 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1136318 T2446 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1136323 T2446 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1136323 T2446 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1136324 T2446 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1136324 T2446 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1136325 T2446 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1136325 T2446 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1136325 T2446 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1136325 T2446 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1136326 T2446 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/control-001/cores/collection1/data/
   [junit4]   2> 1136326 T2446 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@143d13d6
   [junit4]   2> 1136327 T2446 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/control-001/cores/collection1/data
   [junit4]   2> 1136327 T2446 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/control-001/cores/collection1/data/index/
   [junit4]   2> 1136327 T2446 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1136328 T2446 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/control-001/cores/collection1/data/index
   [junit4]   2> 1136328 T2446 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=34, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=57.4541015625, floorSegmentMB=0.5546875, forceMergeDeletesPctAllowed=5.22780806596198, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.19405204540464516
   [junit4]   2> 1136329 T2446 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@34e08c6c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2232193),segFN=segments_1,generation=1}
   [junit4]   2> 1136329 T2446 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1136335 T2446 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1136335 T2446 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1136336 T2446 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1136336 T2446 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1136336 T2446 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1136336 T2446 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1136337 T2446 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1136337 T2446 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1136337 T2446 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1136365 T2446 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1136368 T2446 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1136370 T2446 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1136373 T2446 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1136378 T2446 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/system,/admin/logging,/update/csv,/update,/admin/plugins,/update/json,/admin/file,/get,standard,/admin/mbeans,/replication,/admin/threads,/config,/admin/segments,/admin/luke,/admin/properties,/schema,/admin/ping,/update/json/docs
   [junit4]   2> 1136381 T2446 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1136383 T2446 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1136383 T2446 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1136385 T2446 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1136385 T2446 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1136386 T2446 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=21, maxMergeAtOnceExplicit=36, maxMergedSegmentMB=22.39453125, floorSegmentMB=1.7548828125, forceMergeDeletesPctAllowed=9.803215846184616, segmentsPerTier=36.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1136387 T2446 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@34e08c6c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2232193),segFN=segments_1,generation=1}
   [junit4]   2> 1136387 T2446 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1136387 T2446 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@2929af9d[collection1] main
   [junit4]   2> 1136388 T2446 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1136388 T2446 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1136389 T2446 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1136389 T2446 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1136389 T2446 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1136390 T2446 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1136390 T2446 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1136390 T2446 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1136391 T2446 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1136391 T2446 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1136392 T2447 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2929af9d[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1136392 T2446 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1136393 T2450 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:50207/zt_/rp collection:control_collection shard:shard1
   [junit4]   2> 1136393 T2416 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 1136394 T2416 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1136394 T2450 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1136396 T2416 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1136396 T2416 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1136398 T2454 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@688cf5a5 name:ZooKeeperConnection Watcher:127.0.0.1:50202/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1136398 T2416 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1136398 T2450 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1136398 T2416 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1136399 T2416 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1136400 T2442 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1136401 T2450 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1136401 T2450 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C428 name=collection1 org.apache.solr.core.SolrCore@115569db url=http://127.0.0.1:50207/zt_/rp/collection1 node=127.0.0.1:50207_zt_%2Frp C428_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:50207/zt_/rp, core=collection1, node_name=127.0.0.1:50207_zt_%2Frp, state=down}
   [junit4]   2> 1136401 T2450 C:control_collection S:shard1 C428 P50207 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:50207/zt_/rp/collection1/
   [junit4]   2> 1136401 T2443 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 1136402 T2416 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1136401 T2450 C:control_collection S:shard1 C428 P50207 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1136402 T2416 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 1136402 T2450 C:control_collection S:shard1 C428 P50207 oasc.SyncStrategy.syncToMe http://127.0.0.1:50207/zt_/rp/collection1/ has no replicas
   [junit4]   2> 1136403 T2442 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1136403 T2450 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:50207/zt_/rp/collection1/ shard1
   [junit4]   2> 1136403 T2454 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1136402 T2416 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1136403 T2450 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1136404 T2416 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1136406 T2457 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@634940ba name:ZooKeeperConnection Watcher:127.0.0.1:50202/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1136406 T2416 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1136406 T2416 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1136408 T2442 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1136409 T2443 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 2, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 1136410 T2443 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 1136412 T2443 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50207/zt_/rp",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 1136412 T2443 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 1136413 T2454 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1136413 T2442 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1136416 T2442 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1136416 T2442 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1136416 T2454 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1136460 T2450 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:50207/zt_/rp/collection1/ and leader is http://127.0.0.1:50207/zt_/rp/collection1/
   [junit4]   2> 1136461 T2450 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:50207/zt_/rp
   [junit4]   2> 1136461 T2450 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1136461 T2450 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1136461 T2450 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1136463 T2442 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1136464 T2443 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50207/zt_/rp",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:50207_zt_%2Frp",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":"shard1"} current state version: 4
   [junit4]   2> 1136464 T2443 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50207/zt_/rp",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:50207_zt_%2Frp",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":"shard1"}
   [junit4]   2> 1136568 T2442 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1136568 T2454 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1137646 T2416 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001/cores/collection1
   [junit4]   2> 1137648 T2416 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001
   [junit4]   2> 1137649 T2416 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1137651 T2416 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:50220
   [junit4]   2> 1137652 T2416 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/zt_/rp, solrconfig=solrconfig.xml, hostPort=50219, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/tempDir-001/jetty1, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001/cores}
   [junit4]   2> 1137652 T2416 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 1137652 T2416 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001/'
   [junit4]   2> 1137681 T2416 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001/solr.xml
   [junit4]   2> 1137716 T2416 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001/cores
   [junit4]   2> 1137717 T2416 oasc.CoreContainer.<init> New CoreContainer 294604378
   [junit4]   2> 1137717 T2416 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001/]
   [junit4]   2> 1137717 T2416 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001/lib
   [junit4]   2> 1137718 T2416 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001/lib).
   [junit4]   2> 1137727 T2416 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1137727 T2416 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1137727 T2416 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1137728 T2416 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1137728 T2416 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1137728 T2416 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1137728 T2416 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1137728 T2416 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1137729 T2416 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1137729 T2416 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1137729 T2416 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1137734 T2416 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1137734 T2416 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1137734 T2416 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1137735 T2416 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1137735 T2416 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50202/solr
   [junit4]   2> 1137735 T2416 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1137735 T2416 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1137736 T2416 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1137738 T2469 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3d722847 name:ZooKeeperConnection Watcher:127.0.0.1:50202 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1137738 T2416 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1137739 T2416 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1137740 T2416 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1137741 T2472 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@116eb63 name:ZooKeeperConnection Watcher:127.0.0.1:50202/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1137742 T2416 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1137745 T2416 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1138749 T2416 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50219_zt_%2Frp
   [junit4]   2> 1138750 T2416 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50219_zt_%2Frp
   [junit4]   2> 1138753 T2416 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1138756 T2416 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001/cores
   [junit4]   2> 1138758 T2416 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001/cores/collection1/, dataDir=data/, transient=false, loadOnStartup=true, config=solrconfig.xml, coreNodeName=, collection=collection1, shard=, name=collection1, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001/cores/collection1}
   [junit4]   2> 1138759 T2416 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001/cores/collection1/
   [junit4]   2> 1138759 T2416 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1138761 T2473 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1138761 T2473 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1138762 T2442 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1138762 T2473 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 1138763 T2473 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 1138763 T2443 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50219/zt_/rp",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:50219_zt_%2Frp",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null} current state version: 5
   [junit4]   2> 1138763 T2473 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 1138763 T2443 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50219/zt_/rp",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:50219_zt_%2Frp",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 1138764 T2443 oasco.ReplicaMutator.updateState Collection already exists with numShards=3
   [junit4]   2> 1138764 T2443 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1138764 T2473 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 1138764 T2473 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1138765 T2443 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 1138765 T2472 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1138766 T2472 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 1139765 T2473 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1139766 T2473 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1139766 T2473 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1139766 T2473 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1139767 T2473 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1139767 T2473 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001/cores/collection1/'
   [junit4]   2> 1139780 T2473 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1139782 T2473 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1139791 T2473 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1139810 T2473 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1139851 T2473 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1139952 T2473 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1139959 T2473 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1140253 T2473 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1140263 T2473 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1140266 T2473 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1140272 T2473 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1140275 T2473 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1140278 T2473 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1140279 T2473 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1140279 T2473 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1140279 T2473 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1140279 T2473 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1140280 T2473 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1140280 T2473 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1140280 T2473 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1140280 T2473 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 1140280 T2473 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@143d13d6
   [junit4]   2> 1140281 T2473 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001/cores/collection1/data
   [junit4]   2> 1140281 T2473 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 1140282 T2473 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1140282 T2473 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest A6DBB2D7042F4A95-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 1140282 T2473 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=34, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=57.4541015625, floorSegmentMB=0.5546875, forceMergeDeletesPctAllowed=5.22780806596198, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.19405204540464516
   [junit4]   2> 1140283 T2473 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5eff06be lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3913d4ab),segFN=segments_1,generation=1}
   [junit4]   2> 1140283 T2473 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1140288 T2473 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1140288 T2473 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1140288 T2473 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1140288 T2473 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1140288 T2473 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1140288 T2473 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1140289 T2473 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1140289 T2473 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1140289 T2473 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1140301 T2473 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1140303 T2473 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1140304 T2473 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1140306 T2473 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1140311 T2473 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/system,/admin/logging,/update/csv,/update,/admin/plugins,/update/json,/admin/file,/get,standard,/admin/mbeans,/replication,/admin/threads,/config,/admin/segments,/admin/luke,/admin/properties,/schema,/admin/ping,/update/json/docs
   [junit4]   2> 1140312 T2473 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1140313 T2473 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1140313 T2473 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1140314 T2473 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1140314 T2473 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1140315 T2473 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=21, maxMergeAtOnceExplicit=36, maxMergedSegmentMB=22.39453125, floorSegmentMB=1.7548828125, forceMergeDeletesPctAllowed=9.803215846184616, segmentsPerTier=36.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1140316 T2473 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5eff06be lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3913d4ab),segFN=segments_1,generation=1}
   [junit4]   2> 1140316 T2473 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1140316 T2473 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@1857ee1b[collection1] main
   [junit4]   2> 1140316 T2473 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1140317 T2473 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1140317 T2473 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1140317 T2473 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1140317 T2473 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1140318 T2473 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1140318 T2473 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1140318 T2473 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1140318 T2473 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1140319 T2473 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1140319 T2474 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1857ee1b[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1140320 T2473 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1140321 T2477 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:50219/zt_/rp collection:collection1 shard:shard1
   [junit4]   2> 1140321 T2416 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 1140321 T2416 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1140321 T2477 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1140324 T2477 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1140325 T2442 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1140326 T2477 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1140326 T2477 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C429 name=collection1 org.apache.solr.core.SolrCore@796050a8 url=http://127.0.0.1:50219/zt_/rp/collection1 node=127.0.0.1:50219_zt_%2Frp C429_STATE=coll:collection1 core:collection1 props:{base_url=http://127.0.0.1:50219/zt_/rp, core=collection1, node_name=127.0.0.1:50219_zt_%2Frp, state=down}
   [junit4]   2> 1140326 T2477 C:collection1 S:shard1 C429 P50219 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:50219/zt_/rp/collection1/
   [junit4]   2> 1140326 T2443 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 1140326 T2477 C:collection1 S:shard1 C429 P50219 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1140327 T2477 C:collection1 S:shard1 C429 P50219 oasc.SyncStrategy.syncToMe http://127.0.0.1:50219/zt_/rp/collection1/ has no replicas
   [junit4]   2> 1140327 T2477 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:50219/zt_/rp/collection1/ shard1
   [junit4]   2> 1140327 T2477 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1140327 T2443 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 1140328 T2472 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1140328 T2472 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 1140330 T2443 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50219/zt_/rp",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 1140330 T2443 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 1140331 T2472 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1140331 T2472 oascc.ZkStateRea

[...truncated too long message...]

 		4	/solr/aliases.json
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 		2	/solr/collections/collection1/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1598486 T4791 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:48552/
   [junit4]   2> 1598486 T4791 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:48526/
   [junit4]   2> 1598486 T4791 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:48539/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=A6DBB2D7042F4A95 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=nl_NL -Dtests.timezone=Asia/Pyongyang -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   66.5s J0 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48539/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([A6DBB2D7042F4A95:2E8F8D0DAAD3276D]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48539/collection1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 49 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:48539 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 54 more
   [junit4]   2> 1598502 T4791 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest A6DBB2D7042F4A95-001
   [junit4]   2> 66505 T4790 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {a_t=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), text=PostingsFormat(name=Memory doPackFST= true), intDefault=PostingsFormat(name=LuceneFixedGap), id=Lucene50(blocksize=128), range_facet_l=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), _version_=PostingsFormat(name=LuceneFixedGap), timestamp=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), multiDefault=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128)))}, docValues:{timestamp=DocValuesFormat(name=Asserting)}, sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {}, locale=nl_NL, timezone=Asia/Pyongyang
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=96816168,total=301465600
   [junit4]   2> NOTE: All tests run in this JVM: [PluginInfoTest, SearchHandlerTest, TestNoOpRegenerator, HdfsSyncSliceTest, TestZkChroot, TestElisionMultitermQuery, CoreAdminHandlerTest, HttpPartitionTest, VMParamsZkACLAndCredentialsProvidersTest, SpellCheckCollatorTest, DistanceFunctionTest, TestRTGBase, TestCSVResponseWriter, TestShardHandlerFactory, MigrateRouteKeyTest, TestDefaultStatsCache, TestSolrXml, RequestLoggingTest, TestConfigReload, DirectSolrSpellCheckerTest, TestSortingResponseWriter, DistributedQueueTest, EchoParamsTest, DOMUtilTest, CircularListTest, TestClassNameShortening, ChangedSchemaMergeTest, TestSweetSpotSimilarityFactory, ActionThrottleTest, TestFastOutputStream, CustomCollectionTest, TermVectorComponentDistributedTest, SpellCheckComponentTest, TestDistributedMissingSort, CacheHeaderTest, MultiTermTest, TestBulkSchemaAPI, TermVectorComponentTest, TestSolrIndexConfig, TestSystemIdResolver, CurrencyFieldOpenExchangeTest, BlockDirectoryTest, DeleteReplicaTest, LoggingHandlerTest, ReplicationFactorTest, TestExactSharedStatsCache, TestSimpleQParserPlugin, TestMissingGroups, TestStressLucene, SolrXmlInZkTest, SpatialHeatmapFacetsTest, UpdateRequestProcessorFactoryTest, SolrCoreTest, SharedFSAutoReplicaFailoverTest, SolrTestCaseJ4Test, TestLeaderElectionZkExpiry, TestNamedUpdateProcessors, MultiThreadedOCPTest, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J0 in 67.04s, 1 test, 1 error <<< FAILURES!

[...truncated 691 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 54 minutes 6 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 212405397 bytes
Compression is 0.0%
Took 42 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2893 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2893/

5 tests failed.
REGRESSION:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test

Error Message:
There were too many update fails (39 > 20) - we expect it can happen, but shouldn't easily

Stack Trace:
java.lang.AssertionError: There were too many update fails (39 > 20) - we expect it can happen, but shouldn't easily
	at __randomizedtesting.SeedInfo.seed([25E72E934121CB2B:ADB31149EFDDA6D3]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertFalse(Assert.java:68)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:230)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.solr.handler.component.DistributedMLTComponentTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:37002//collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:37002//collection1
	at __randomizedtesting.SeedInfo.seed([25E72E934121CB2B:ADB31149EFDDA6D3]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:566)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:943)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:558)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:606)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:588)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:567)
	at org.apache.solr.handler.component.DistributedMLTComponentTest.test(DistributedMLTComponentTest.java:126)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 51 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:22086/pd/l/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22086/pd/l/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([25E72E934121CB2B:ADB31149EFDDA6D3]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:22086 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:38235/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:38235/collection1
	at __randomizedtesting.SeedInfo.seed([25E72E934121CB2B:ADB31149EFDDA6D3]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:38235/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:38235 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:20826/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:20826/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([25E72E934121CB2B:ADB31149EFDDA6D3]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:20826/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:20826 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9549 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/init-core-data-001
   [junit4]   2> 380058 T1111 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /pd/l
   [junit4]   2> 380066 T1111 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 380066 T1112 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 380166 T1111 oasc.ZkTestServer.run start zk server on port:22062
   [junit4]   2> 380167 T1111 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 380168 T1111 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 380195 T1119 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2f3ded81 name:ZooKeeperConnection Watcher:127.0.0.1:22062 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 380196 T1111 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 380196 T1111 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 380196 T1111 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 380199 T1111 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 380200 T1111 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 380201 T1122 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7337503e name:ZooKeeperConnection Watcher:127.0.0.1:22062/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 380201 T1111 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 380202 T1111 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 380202 T1111 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 380205 T1111 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 380206 T1111 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 380207 T1111 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 380209 T1111 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 380210 T1111 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 380212 T1111 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 380213 T1111 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 380215 T1111 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 380216 T1111 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 380218 T1111 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 380218 T1111 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 380220 T1111 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 380221 T1111 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 380222 T1111 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 380223 T1111 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 380224 T1111 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 380225 T1111 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 380227 T1111 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 380227 T1111 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 380229 T1111 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 380230 T1111 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 380231 T1111 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 380232 T1111 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 380234 T1111 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 380234 T1111 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 381843 T1111 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/control-001/cores/collection1
   [junit4]   2> 381848 T1111 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 381852 T1111 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:22069
   [junit4]   2> 381853 T1111 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/control-001/cores, hostContext=/pd/l, hostPort=22068, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/tempDir-001/control/data}
   [junit4]   2> 381853 T1111 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 381854 T1111 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/control-001/'
   [junit4]   2> 382072 T1111 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/control-001/solr.xml
   [junit4]   2> 382126 T1111 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/control-001/cores
   [junit4]   2> 382126 T1111 oasc.CoreContainer.<init> New CoreContainer 1645308729
   [junit4]   2> 382127 T1111 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/control-001/]
   [junit4]   2> 382127 T1111 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/control-001/lib
   [junit4]   2> 382128 T1111 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/control-001/lib).
   [junit4]   2> 382137 T1111 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 382137 T1111 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 382137 T1111 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 382137 T1111 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 382138 T1111 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 382138 T1111 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 382138 T1111 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 382138 T1111 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 382138 T1111 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 382139 T1111 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 382139 T1111 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 382154 T1111 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 382154 T1111 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 382155 T1111 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 382155 T1111 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 382155 T1111 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:22062/solr
   [junit4]   2> 382155 T1111 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 382155 T1111 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 382156 T1111 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 382158 T1134 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@62e48bd5 name:ZooKeeperConnection Watcher:127.0.0.1:22062 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 382159 T1111 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 382160 T1111 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 382162 T1111 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 382164 T1137 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3f1d425d name:ZooKeeperConnection Watcher:127.0.0.1:22062/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 382164 T1111 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 382166 T1111 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 382168 T1111 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 382170 T1111 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 382173 T1111 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 382175 T1111 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 382177 T1111 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 382179 T1111 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 382180 T1111 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 382181 T1111 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:22068_pd%2Fl
   [junit4]   2> 382182 T1111 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:22068_pd%2Fl
   [junit4]   2> 382184 T1111 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 382185 T1111 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 382187 T1111 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 382188 T1111 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:22068_pd%2Fl
   [junit4]   2> 382189 T1111 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 382190 T1111 oasc.Overseer.start Overseer (id=93599020707741699-127.0.0.1:22068_pd%2Fl-n_0000000000) starting
   [junit4]   2> 382192 T1111 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 382198 T1111 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 382198 T1139 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 382198 T1111 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 382199 T1138 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 382203 T1111 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/control-001/cores
   [junit4]   2> 382206 T1111 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/control-001/cores/collection1/, dataDir=data/, loadOnStartup=true, shard=, collection=control_collection, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/control-001/cores/collection1, transient=false, schema=schema.xml, coreNodeName=, name=collection1}
   [junit4]   2> 382206 T1111 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/control-001/cores/collection1/
   [junit4]   2> 382206 T1111 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 382208 T1141 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 382208 T1141 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 382209 T1137 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 382209 T1141 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 382210 T1138 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22068/pd/l",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:22068_pd%2Fl",
   [junit4]   2> 	  "numShards":"1"} current state version: 0
   [junit4]   2> 382211 T1138 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22068/pd/l",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:22068_pd%2Fl",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 382211 T1138 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 382211 T1138 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 382212 T1137 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 383210 T1141 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 383211 T1141 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 383211 T1141 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 383212 T1141 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 383212 T1141 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 383212 T1141 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/control-001/cores/collection1/'
   [junit4]   2> 383241 T1141 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 383243 T1141 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 383251 T1141 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 383266 T1141 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 383311 T1141 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 383312 T1141 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 383326 T1141 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 383729 T1141 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 383746 T1141 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 383757 T1141 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 383773 T1141 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 383782 T1141 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 383786 T1141 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 383787 T1141 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 383787 T1141 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 383787 T1141 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 383788 T1141 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 383788 T1141 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 383788 T1141 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 383789 T1141 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 383789 T1141 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/control-001/cores/collection1/data/
   [junit4]   2> 383789 T1141 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6ad957a7
   [junit4]   2> 383790 T1141 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/control-001/cores/collection1/data
   [junit4]   2> 383791 T1141 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/control-001/cores/collection1/data/index/
   [junit4]   2> 383791 T1141 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 383791 T1141 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/control-001/cores/collection1/data/index
   [junit4]   2> 383792 T1141 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=33, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 383793 T1141 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5f7ee311 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@36da212),segFN=segments_1,generation=1}
   [junit4]   2> 383793 T1141 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 383798 T1141 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 383799 T1141 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 383799 T1141 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 383799 T1141 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 383800 T1141 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 383800 T1141 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 383800 T1141 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 383801 T1141 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 383801 T1141 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 383817 T1141 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 383823 T1141 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 383825 T1141 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 383826 T1141 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 383830 T1141 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update,/admin/logging,standard,/replication,/update/json,/admin/system,/admin/threads,/schema,/admin/segments,/admin/plugins,/update/csv,/admin/mbeans,/admin/properties,/get,/config,/update/json/docs,/admin/ping,/admin/luke,/admin/file
   [junit4]   2> 383842 T1141 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 383844 T1141 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 383844 T1141 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 383846 T1141 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 383846 T1141 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 383846 T1141 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=25, maxMergedSegmentMB=75.1943359375, floorSegmentMB=1.021484375, forceMergeDeletesPctAllowed=10.720232696799917, segmentsPerTier=23.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8897883915405346
   [junit4]   2> 383847 T1141 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5f7ee311 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@36da212),segFN=segments_1,generation=1}
   [junit4]   2> 383847 T1141 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 383848 T1141 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@632703e[collection1] main
   [junit4]   2> 383848 T1141 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 383849 T1141 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 383849 T1141 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 383849 T1141 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 383850 T1141 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 383850 T1141 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 383850 T1141 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 383851 T1141 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 383851 T1141 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 383851 T1141 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 383852 T1141 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 383852 T1142 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@632703e[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 383853 T1145 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:22068/pd/l collection:control_collection shard:shard1
   [junit4]   2> 383853 T1111 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 383854 T1111 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 383854 T1145 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 383857 T1111 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 383858 T1145 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 383858 T1111 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 383859 T1137 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 383860 T1145 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 383860 T1149 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@298a5b60 name:ZooKeeperConnection Watcher:127.0.0.1:22062/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 383860 T1145 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 383860 T1138 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C274 name=collection1 org.apache.solr.core.SolrCore@293a7fdb url=http://127.0.0.1:22068/pd/l/collection1 node=127.0.0.1:22068_pd%2Fl C274_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:22068/pd/l, core=collection1, node_name=127.0.0.1:22068_pd%2Fl}
   [junit4]   2> 383860 T1145 C:control_collection S:shard1 C274 P22068 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:22068/pd/l/collection1/
   [junit4]   2> 383860 T1111 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 383861 T1145 C:control_collection S:shard1 C274 P22068 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 383861 T1145 C:control_collection S:shard1 C274 P22068 oasc.SyncStrategy.syncToMe http://127.0.0.1:22068/pd/l/collection1/ has no replicas
   [junit4]   2> 383861 T1111 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 383861 T1145 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:22068/pd/l/collection1/ shard1
   [junit4]   2> 383862 T1137 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 383861 T1111 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 383862 T1145 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 383865 T1111 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 383866 T1137 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 383866 T1138 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22068/pd/l",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 383969 T1137 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 383969 T1149 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 384018 T1145 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:22068/pd/l/collection1/ and leader is http://127.0.0.1:22068/pd/l/collection1/
   [junit4]   2> 384019 T1145 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:22068/pd/l
   [junit4]   2> 384019 T1145 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 384019 T1145 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 384019 T1145 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 384021 T1137 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 384022 T1138 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22068/pd/l",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:22068_pd%2Fl",
   [junit4]   2> 	  "numShards":"1"} current state version: 3
   [junit4]   2> 384022 T1138 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22068/pd/l",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:22068_pd%2Fl",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 384125 T1137 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 384125 T1149 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 385499 T1111 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001/cores/collection1
   [junit4]   2> 385504 T1111 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001
   [junit4]   2> 385505 T1111 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 385509 T1111 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:22087
   [junit4]   2> 385510 T1111 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=22086, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001/cores, solrconfig=solrconfig.xml, hostContext=/pd/l, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/tempDir-001/jetty1}
   [junit4]   2> 385510 T1111 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 385511 T1111 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001/'
   [junit4]   2> 385557 T1111 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001/solr.xml
   [junit4]   2> 385618 T1111 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001/cores
   [junit4]   2> 385619 T1111 oasc.CoreContainer.<init> New CoreContainer 1755072217
   [junit4]   2> 385619 T1111 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001/]
   [junit4]   2> 385619 T1111 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001/lib
   [junit4]   2> 385620 T1111 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001/lib).
   [junit4]   2> 385639 T1111 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 385639 T1111 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 385639 T1111 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 385640 T1111 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 385640 T1111 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 385641 T1111 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 385641 T1111 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 385642 T1111 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 385642 T1111 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 385642 T1111 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 385643 T1111 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 385650 T1111 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 385651 T1111 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 385651 T1111 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 385652 T1111 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 385652 T1111 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:22062/solr
   [junit4]   2> 385652 T1111 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 385653 T1111 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 385654 T1111 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 385655 T1161 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@78134d4c name:ZooKeeperConnection Watcher:127.0.0.1:22062 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 385656 T1111 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 385657 T1111 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 385660 T1111 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 385662 T1164 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2f6887b3 name:ZooKeeperConnection Watcher:127.0.0.1:22062/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 385662 T1111 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 385667 T1111 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 386671 T1111 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:22086_pd%2Fl
   [junit4]   2> 386672 T1111 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:22086_pd%2Fl
   [junit4]   2> 386675 T1111 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 386677 T1111 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001/cores
   [junit4]   2> 386681 T1111 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {loadOnStartup=true, coreNodeName=, name=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001/cores/collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001/cores/collection1/, transient=false, shard=, dataDir=data/, collection=collection1, config=solrconfig.xml, schema=schema.xml}
   [junit4]   2> 386681 T1111 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001/cores/collection1/
   [junit4]   2> 386682 T1111 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 386683 T1165 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 386684 T1165 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 386685 T1165 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 386685 T1137 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 386686 T1138 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22086/pd/l",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:22086_pd%2Fl",
   [junit4]   2> 	  "numShards":"1"} current state version: 4
   [junit4]   2> 386686 T1138 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22086/pd/l",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:22086_pd%2Fl",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 386686 T1138 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 386687 T1138 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 386688 T1164 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 386688 T1149 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 386688 T1137 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 387685 T1165 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 387686 T1165 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 387686 T1165 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 387687 T1165 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 387687 T1165 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 387688 T1165 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001/cores/collection1/'
   [junit4]   2> 387705 T1165 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 387707 T1165 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 387719 T1165 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 387745 T1165 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 387798 T1165 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 387799 T1165 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 387819 T1165 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 388122 T1165 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 388140 T1165 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 388142 T1165 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 388151 T1165 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 388154 T1165 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 388157 T1165 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 388158 T1165 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 388158 T1165 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 388158 T1165 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 388159 T1165 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 388159 T1165 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 388159 T1165 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 388159 T1165 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 388159 T1165 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 388160 T1165 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6ad957a7
   [junit4]   2> 388160 T1165 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001/cores/collection1/data
   [junit4]   2> 388161 T1165 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 388161 T1165 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 388161 T1165 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 25E72E934121CB2B-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 388161 T1165 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=33, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 388162 T1165 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@542169c0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5da5e766),segFN=segments_1,generation=1}
   [junit4]   2> 388162 T1165 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 388167 T1165 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init cr

[...truncated too long message...]

934121CB2B-001/tempDir-001/shard2/cores/collection1/data;done=false>>]
   [junit4]   2> 2195371 T6981 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 25E72E934121CB2B-001/tempDir-001/shard2/cores/collection1/data
   [junit4]   2> 2195372 T6981 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 25E72E934121CB2B-001/tempDir-001/shard2/cores/collection1/data/index [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 25E72E934121CB2B-001/tempDir-001/shard2/cores/collection1/data/index;done=false>>]
   [junit4]   2> 2195372 T6981 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 25E72E934121CB2B-001/tempDir-001/shard2/cores/collection1/data/index
   [junit4]   2> 2195373 T6981 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=DistributedMLTComponentTest -Dtests.method=test -Dtests.seed=25E72E934121CB2B -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=sr_CS -Dtests.timezone=Etc/GMT+1 -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR    102s J3 | DistributedMLTComponentTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:37002//collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([25E72E934121CB2B:ADB31149EFDDA6D3]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:566)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:943)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:558)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:606)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:588)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:567)
   [junit4]    > 	at org.apache.solr.handler.component.DistributedMLTComponentTest.test(DistributedMLTComponentTest.java:126)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: java.net.SocketTimeoutException: Read timed out
   [junit4]    > 	at java.net.SocketInputStream.socketRead0(Native Method)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:152)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:122)
   [junit4]    > 	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
   [junit4]    > 	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
   [junit4]    > 	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 51 more
   [junit4]   2> 2195581 T6981 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 25E72E934121CB2B-001
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarityProvider(queryNorm=false,coord=no): {}, locale=sr_CS, timezone=Etc/GMT+1
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=229360800,total=411566080
   [junit4]   2> NOTE: All tests run in this JVM: [EchoParamsTest, RequestHandlersTest, TestIndexingPerformance, ShowFileRequestHandlerTest, BasicFunctionalityTest, StatsComponentTest, IndexSchemaRuntimeFieldTest, TestSort, CoreMergeIndexesAdminHandlerTest, TestCloudInspectUtil, PrimUtilsTest, TestSearchPerf, TestSolrDeletionPolicy2, LeaderElectionIntegrationTest, BadIndexSchemaTest, AtomicUpdatesTest, TestManagedSchemaDynamicFieldResource, TestRealTimeGet, TestReversedWildcardFilterFactory, TestExtendedDismaxParser, BasicDistributedZkTest, TestHdfsUpdateLog, TestSerializedLuceneMatchVersion, BasicDistributedZk2Test, TestCloudManagedSchema, TestSolrJ, TestQuerySenderNoQuery, TestStressVersions, SimpleCollectionCreateDeleteTest, TestLRUStatsCache, HdfsDirectoryFactoryTest, DistributedTermsComponentTest, WordBreakSolrSpellCheckerTest, TestFoldingMultitermQuery, DistributedSpellCheckComponentTest, TestBlendedInfixSuggestions, TestIndexSearcher, DocValuesMissingTest, AsyncMigrateRouteKeyTest, TestManagedSynonymFilterFactory, QueryEqualityTest, CollectionsAPIAsyncDistributedZkTest, TestManagedSchema, JSONWriterTest, TestDocumentBuilder, ChangedSchemaMergeTest, TimeZoneUtilsTest, MergeStrategyTest, HighlighterTest, TestRemoteStreaming, DateMathParserTest, TestEmbeddedSolrServerConstructors, SolrInfoMBeanTest, TestHashPartitioner, CloudExitableDirectoryReaderTest, TestLuceneMatchVersion, CollectionsAPIDistributedZkTest, TestCloudManagedSchemaConcurrent, TestCloudSchemaless, UnloadDistributedZkTest, TestBlobHandler, MigrateRouteKeyTest, DistribJoinFromCollectionTest, ChaosMonkeySafeLeaderTest, TestDistribDocBasedVersion, DistributedFacetPivotLargeTest, TestDistributedMissingSort, DistributedFacetPivotWhiteBoxTest, DistributedMLTComponentTest]
   [junit4] Completed on J3 in 102.46s, 1 test, 1 error <<< FAILURES!

[...truncated 525 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 4 errors, 1 failure, 61 ignored (21 assumptions)

Total time: 58 minutes 17 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 226560664 bytes
Compression is 0.0%
Took 46 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2892 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2892/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:59918/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:59918/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([E99E0FB8B730BCED:61CA306219CCD115]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:59918 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:43674/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:43674/collection1
	at __randomizedtesting.SeedInfo.seed([E99E0FB8B730BCED:61CA306219CCD115]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:43674/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:43674 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:59962/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:59962/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([E99E0FB8B730BCED:61CA306219CCD115]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:59962/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:59962 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 10024 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/init-core-data-001
   [junit4]   2> 1396045 T5918 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 1396049 T5918 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1396050 T5919 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1396150 T5918 oasc.ZkTestServer.run start zk server on port:59906
   [junit4]   2> 1396151 T5918 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1396152 T5918 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1396155 T5926 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3b5e611f name:ZooKeeperConnection Watcher:127.0.0.1:59906 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1396155 T5918 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1396155 T5918 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1396156 T5918 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1396157 T5918 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1396158 T5918 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1396160 T5929 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@40502e3a name:ZooKeeperConnection Watcher:127.0.0.1:59906/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1396160 T5918 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1396160 T5918 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1396160 T5918 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1396162 T5918 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1396163 T5918 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1396164 T5918 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1396165 T5918 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1396166 T5918 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1396168 T5918 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1396168 T5918 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1396271 T5918 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1396271 T5918 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1396273 T5918 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1396273 T5918 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1396274 T5918 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1396275 T5918 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1396276 T5918 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1396277 T5918 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1396278 T5918 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1396278 T5918 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1396280 T5918 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1396280 T5918 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1396281 T5918 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1396282 T5918 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1396283 T5918 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1396284 T5918 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1396285 T5918 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1396285 T5918 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1397467 T5918 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/control-001/cores/collection1
   [junit4]   2> 1397470 T5918 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1397474 T5918 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:59911
   [junit4]   2> 1397474 T5918 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/control-001/cores, hostContext=/, hostPort=59910}
   [junit4]   2> 1397475 T5918 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 1397475 T5918 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/control-001/'
   [junit4]   2> 1397510 T5918 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/control-001/solr.xml
   [junit4]   2> 1397558 T5918 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/control-001/cores
   [junit4]   2> 1397558 T5918 oasc.CoreContainer.<init> New CoreContainer 806518256
   [junit4]   2> 1397558 T5918 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/control-001/]
   [junit4]   2> 1397559 T5918 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/control-001/lib
   [junit4]   2> 1397559 T5918 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/control-001/lib).
   [junit4]   2> 1397569 T5918 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1397569 T5918 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1397570 T5918 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1397570 T5918 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1397570 T5918 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1397570 T5918 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1397570 T5918 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1397571 T5918 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1397571 T5918 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1397571 T5918 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1397571 T5918 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1397579 T5918 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1397579 T5918 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1397580 T5918 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1397580 T5918 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1397580 T5918 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:59906/solr
   [junit4]   2> 1397580 T5918 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1397581 T5918 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1397581 T5918 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1397583 T5941 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3e2ee70f name:ZooKeeperConnection Watcher:127.0.0.1:59906 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1397583 T5918 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1397583 T5918 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1397585 T5918 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1397586 T5944 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@280478a3 name:ZooKeeperConnection Watcher:127.0.0.1:59906/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1397586 T5918 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1397587 T5918 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1397588 T5918 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1397590 T5918 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1397591 T5918 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1397593 T5918 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1397594 T5918 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1397596 T5918 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1397597 T5918 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1397597 T5918 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59910_
   [junit4]   2> 1397598 T5918 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:59910_
   [junit4]   2> 1397599 T5918 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1397600 T5918 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1397601 T5918 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1397602 T5918 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:59910_
   [junit4]   2> 1397602 T5918 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1397603 T5918 oasc.Overseer.start Overseer (id=93598135630954499-127.0.0.1:59910_-n_0000000000) starting
   [junit4]   2> 1397605 T5918 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1397609 T5918 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 1397610 T5946 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1397610 T5918 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1397610 T5945 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1397613 T5918 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/control-001/cores
   [junit4]   2> 1397615 T5918 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {coreNodeName=, dataDir=data/, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/control-001/cores/collection1, schema=schema.xml, loadOnStartup=true, shard=, name=collection1, transient=false, collection=control_collection, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/control-001/cores/collection1/}
   [junit4]   2> 1397615 T5918 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/control-001/cores/collection1/
   [junit4]   2> 1397616 T5918 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1397616 T5948 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1397616 T5948 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1397617 T5948 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1397617 T5944 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1397618 T5945 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59910",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:59910_",
   [junit4]   2> 	  "roles":null} current state version: 0
   [junit4]   2> 1397618 T5945 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59910",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:59910_",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 1397619 T5945 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 1397619 T5945 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1397620 T5944 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1398617 T5948 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1398618 T5948 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1398618 T5948 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1398618 T5948 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1398619 T5948 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1398619 T5948 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/control-001/cores/collection1/'
   [junit4]   2> 1398631 T5948 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1398633 T5948 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1398640 T5948 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1398656 T5948 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1398715 T5948 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1398817 T5948 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1398825 T5948 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1399173 T5948 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1399187 T5948 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1399189 T5948 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1399196 T5948 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1399200 T5948 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1399202 T5948 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1399203 T5948 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1399203 T5948 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1399203 T5948 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1399204 T5948 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1399204 T5948 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1399204 T5948 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1399204 T5948 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1399205 T5948 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/control-001/cores/collection1/data/
   [junit4]   2> 1399205 T5948 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3fb7bec8
   [junit4]   2> 1399206 T5948 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/control-001/cores/collection1/data
   [junit4]   2> 1399206 T5948 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/control-001/cores/collection1/data/index/
   [junit4]   2> 1399206 T5948 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1399206 T5948 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/control-001/cores/collection1/data/index
   [junit4]   2> 1399207 T5948 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=29, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8275799343616196]
   [junit4]   2> 1399207 T5948 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5d65f114 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4fb687f3),segFN=segments_1,generation=1}
   [junit4]   2> 1399207 T5948 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1399211 T5948 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1399212 T5948 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1399212 T5948 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1399212 T5948 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1399212 T5948 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1399212 T5948 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1399213 T5948 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1399213 T5948 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1399213 T5948 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1399228 T5948 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1399230 T5948 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1399232 T5948 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1399233 T5948 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1399237 T5948 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/mbeans,/get,/update,/admin/segments,/admin/ping,/admin/properties,/update/json,/update/json/docs,/replication,/admin/threads,/update/csv,/admin/logging,/config,/admin/file,standard,/admin/luke,/admin/plugins,/schema,/admin/system
   [junit4]   2> 1399239 T5948 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1399240 T5948 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1399240 T5948 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1399241 T5948 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1399241 T5948 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1399242 T5948 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=923759317, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1399242 T5948 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5d65f114 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4fb687f3),segFN=segments_1,generation=1}
   [junit4]   2> 1399243 T5948 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1399243 T5948 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@1688003a[collection1] main
   [junit4]   2> 1399243 T5948 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1399244 T5948 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1399244 T5948 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1399244 T5948 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1399245 T5948 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1399245 T5948 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1399245 T5948 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1399246 T5948 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1399246 T5948 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1399246 T5948 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1399247 T5949 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1688003a[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1399247 T5948 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1399248 T5952 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:59910 collection:control_collection shard:shard1
   [junit4]   2> 1399248 T5918 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 1399248 T5918 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1399249 T5952 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1399251 T5952 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1399252 T5944 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1399253 T5952 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1399252 T5918 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1399253 T5952 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 1399253 T5945 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 1399253 T5918 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> ASYNC  NEW_CORE C840 name=collection1 org.apache.solr.core.SolrCore@23476853 url=http://127.0.0.1:59910/collection1 node=127.0.0.1:59910_ C840_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:59910, core=collection1, node_name=127.0.0.1:59910_}
   [junit4]   2> 1399253 T5952 C:control_collection S:shard1 C840 P59910 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:59910/collection1/
   [junit4]   2> 1399254 T5952 C:control_collection S:shard1 C840 P59910 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1399254 T5952 C:control_collection S:shard1 C840 P59910 oasc.SyncStrategy.syncToMe http://127.0.0.1:59910/collection1/ has no replicas
   [junit4]   2> 1399254 T5952 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:59910/collection1/ shard1
   [junit4]   2> 1399255 T5952 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1399255 T5944 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1399255 T5956 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@668a2c0b name:ZooKeeperConnection Watcher:127.0.0.1:59906/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1399255 T5918 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1399256 T5918 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1399256 T5918 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1399258 T5944 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1399258 T5918 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1399258 T5945 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59910",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 1399361 T5956 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1399361 T5944 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1399410 T5952 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:59910/collection1/ and leader is http://127.0.0.1:59910/collection1/
   [junit4]   2> 1399411 T5952 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:59910
   [junit4]   2> 1399411 T5952 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1399411 T5952 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1399411 T5952 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1399412 T5944 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1399413 T5945 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59910",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:59910_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1"} current state version: 3
   [junit4]   2> 1399414 T5945 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59910",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:59910_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 1399516 T5956 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1399516 T5944 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1400340 T5918 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001/cores/collection1
   [junit4]   2> 1400342 T5918 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001
   [junit4]   2> 1400343 T5918 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1400345 T5918 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:59927
   [junit4]   2> 1400345 T5918 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostPort=59926, hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001/cores}
   [junit4]   2> 1400346 T5918 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 1400346 T5918 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001/'
   [junit4]   2> 1400377 T5918 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001/solr.xml
   [junit4]   2> 1400415 T5918 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001/cores
   [junit4]   2> 1400416 T5918 oasc.CoreContainer.<init> New CoreContainer 477898052
   [junit4]   2> 1400416 T5918 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001/]
   [junit4]   2> 1400416 T5918 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001/lib
   [junit4]   2> 1400417 T5918 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001/lib).
   [junit4]   2> 1400426 T5918 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1400427 T5918 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1400427 T5918 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1400427 T5918 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1400427 T5918 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1400428 T5918 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1400428 T5918 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1400428 T5918 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1400428 T5918 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1400428 T5918 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1400429 T5918 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1400434 T5918 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1400435 T5918 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1400435 T5918 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1400435 T5918 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1400436 T5918 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:59906/solr
   [junit4]   2> 1400436 T5918 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1400436 T5918 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1400437 T5918 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1400438 T5968 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1b8bb05d name:ZooKeeperConnection Watcher:127.0.0.1:59906 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1400439 T5918 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1400439 T5918 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1400440 T5918 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1400441 T5971 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@d77b265 name:ZooKeeperConnection Watcher:127.0.0.1:59906/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1400441 T5918 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1400444 T5918 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1401447 T5918 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59926_
   [junit4]   2> 1401448 T5918 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:59926_
   [junit4]   2> 1401450 T5918 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1401452 T5918 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001/cores
   [junit4]   2> 1401455 T5918 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {transient=false, shard=, name=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001/cores/collection1, config=solrconfig.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001/cores/collection1/, collection=collection1, coreNodeName=, dataDir=data/, loadOnStartup=true, schema=schema.xml}
   [junit4]   2> 1401455 T5918 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001/cores/collection1/
   [junit4]   2> 1401455 T5918 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1401456 T5972 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1401456 T5972 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1401457 T5972 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1401457 T5944 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1401458 T5945 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59926",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:59926_",
   [junit4]   2> 	  "roles":null} current state version: 4
   [junit4]   2> 1401459 T5945 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59926",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:59926_",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 1401459 T5945 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 1401459 T5945 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1401460 T5944 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1401460 T5971 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1401460 T5956 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1402457 T5972 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1402458 T5972 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1402458 T5972 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1402459 T5972 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1402459 T5972 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1402459 T5972 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001/cores/collection1/'
   [junit4]   2> 1402472 T5972 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1402474 T5972 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1402482 T5972 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1402500 T5972 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1402540 T5972 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1402642 T5972 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1402653 T5972 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1403092 T5972 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1403109 T5972 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1403113 T5972 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1403121 T5972 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1403126 T5972 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1403130 T5972 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1403131 T5972 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1403131 T5972 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1403131 T5972 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1403132 T5972 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1403132 T5972 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1403132 T5972 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1403133 T5972 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1403133 T5972 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 1403133 T5972 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3fb7bec8
   [junit4]   2> 1403134 T5972 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001/cores/collection1/data
   [junit4]   2> 1403135 T5972 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 1403135 T5972 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1403135 T5972 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E99E0FB8B730BCED-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 1403136 T5972 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=29, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8275799343616196]
   [junit4]   2> 1403136 T5972 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@20c962d5 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5779ec4),segFN=segments_1,generation=1}
   [junit4]   2> 1403137 T5972 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1403143 T5972 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1403143 T5972 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1403144 T5972 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1403144 T5972 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1403144 T5972 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1403144 T5972 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1403144 T5972 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1403145 T5972 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1403145 T5972 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1403162 T5972 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1403163 T5972 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1403165 T5972 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1403167 T5972 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1403173 T5972 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/mbeans,/get,/update,/admin/segments,/admin/ping,/admin/properties,/update/json,/update/json/docs,/replication,/admin/threads,/update/csv,/admin/logging,/config,/admin/file,standard,/admin/luke,/admin/plugins,/schema,/admin/system
   [junit4]   2> 1403174 T5972 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1403176 T5972 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1403176 T5972 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1403177 T5972 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1403177 T5972 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1403179 T5972 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=923759317, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1403179 T5972 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@20c962d5 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5779ec4),segFN=segments_1,generation=1}
   [junit4]   2> 1403180 T5972 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1403180 T5972 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@b85b05d[collection1] main
   [junit4]   2> 1403180 T5972 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1403181 T5972 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1403181 T5972 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1403181 T5972 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1403181 T5972 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1403182 T5972 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1403182 T5972 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1403182 T5972 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1403182 T5972 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1403182 T5972 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1403183 T5973 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@b85b05d[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1403184 T5972 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1403184 T5976 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:59926 collection:collection1 shard:shard1
   [junit4]   2> 1403185 T5918 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 1403185 T5918 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1403185 T5976 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1403188 T5976 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1403189 T5944 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1403189 T5976 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1403190 T5976 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C841 name=collection1 org.apache.solr.core.SolrCore@a4ef1e7 url=http://127.0.0.1:59926/collection1 node=127.0.0.1:59926_ C841_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:59926, core=collection1, node_name=127.0.0.1:59926_}
   [junit4]   2> 1403190 T5976 C:collection1 S:shard1 C841 P59926 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:59926/collection1/
   [junit4]   2> 1403190 T5945 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 1403190 T5976 C:collection1 S:shard1 C841 P59926 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1403190 T5976 C:collection1 S:shard1 C841 P59926 oasc.SyncStrategy.syncToMe http://127.0.0.1:59926/collection1/ has no replicas
   [junit4]   2> 1403191 T5976 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:59926/collection1/ shard1
   [junit4]   2> 1403191 T5976 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1403191 T5971 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1403191 T5944 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1403191 T5956 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1403194 T5944 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1403194 T5945 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59926",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 1403297 T5944 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1403297 T5971 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1403297 T5956 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1403346 T5976 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:59926/collection1/ and leader is http://127.0.0.1:59926/collection1/
   [junit4]   2> 1403347 T5976 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:59926
   [junit4]   2> 1403347 T5976 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1403347 T5976 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 1403347 T5976 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1403348 T5944 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1403349 T5945 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59926",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:59926_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1"} current state version: 7
   [junit4]   2> 1403349 T5945 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59926",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:59926_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 1403452 T5956 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1403452 T5944 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1403452 T5971 oascc.ZkStateRe

[...truncated too long message...]

it violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1581182 T6268 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:43674/
   [junit4]   2> 1581182 T6268 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:28515/
   [junit4]   2> 1581183 T6268 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:32427/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=E99E0FB8B730BCED -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=es_VE -Dtests.timezone=WET -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   69.2s J2 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:43674/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([E99E0FB8B730BCED:61CA306219CCD115]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:43674/collection1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 49 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:43674 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 54 more
   [junit4]   2> 1581191 T6268 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest E99E0FB8B730BCED-001
   [junit4]   2> 69248 T6267 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {a_t=PostingsFormat(name=LuceneVarGapFixedInterval), multiDefault=PostingsFormat(name=LuceneVarGapFixedInterval), _version_=PostingsFormat(name=Memory doPackFST= false), id=PostingsFormat(name=Memory doPackFST= true), text=PostingsFormat(name=Direct), intDefault=PostingsFormat(name=Memory doPackFST= false), range_facet_l=PostingsFormat(name=LuceneVarGapFixedInterval), timestamp=PostingsFormat(name=LuceneVarGapFixedInterval)}, docValues:{timestamp=DocValuesFormat(name=Direct)}, sim=DefaultSimilarity, locale=es_VE, timezone=WET
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=237957272,total=354942976
   [junit4]   2> NOTE: All tests run in this JVM: [TestFieldTypeResource, CustomCollectionTest, TestHighFrequencyDictionaryFactory, TestSearchPerf, ConvertedLegacyTest, TestManagedSchema, OpenExchangeRatesOrgProviderTest, TestElisionMultitermQuery, DeleteReplicaTest, DistribDocExpirationUpdateProcessorTest, TestCloudInspectUtil, SearchHandlerTest, BasicZkTest, XmlUpdateRequestHandlerTest, DistributedQueryElevationComponentTest, RecoveryZkTest, TestConfigSets, BlockDirectoryTest, TestCollationField, OverseerTest, TestCoreContainer, TestManagedSchemaDynamicFieldResource, TestSurroundQueryParser, TestUtils, TestCopyFieldCollectionResource, ParsingFieldUpdateProcessorsTest, TestDocumentBuilder, TestRecoveryHdfs, AsyncMigrateRouteKeyTest, SolrXmlInZkTest, TestJmxMonitoredMap, HighlighterMaxOffsetTest, ZkNodePropsTest, TestOrdValues, FieldAnalysisRequestHandlerTest, AddSchemaFieldsUpdateProcessorFactoryTest, RequestLoggingTest, TestFunctionQuery, TestStressVersions, TestCryptoKeys, DistanceFunctionTest, MergeStrategyTest, DisMaxRequestHandlerTest, TestFileDictionaryLookup, CursorPagingTest, SortByFunctionTest, UUIDFieldTest, MultiTermTest, TestBinaryField, TestManagedSchemaFieldResource, TestSolrDeletionPolicy2, TestQueryUtils, TestPseudoReturnFields, TestFieldResource, SyncSliceTest, ZkControllerTest, ResponseHeaderTest, TestSolrXml, TestJoin, TestAddFieldRealTimeGet, TestBadConfig, TestReplicationHandler, FullSolrCloudDistribCmdsTest, ReplicationFactorTest, TestBlobHandler, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J2 in 70.03s, 1 test, 1 error <<< FAILURES!

[...truncated 701 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 54 minutes 34 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 231928266 bytes
Compression is 0.0%
Took 46 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2891 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2891/

4 tests failed.
REGRESSION:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test

Error Message:
There were too many update fails (31 > 20) - we expect it can happen, but shouldn't easily

Stack Trace:
java.lang.AssertionError: There were too many update fails (31 > 20) - we expect it can happen, but shouldn't easily
	at __randomizedtesting.SeedInfo.seed([59822DFF3468BF56:D1D612259A94D2AE]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertFalse(Assert.java:68)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:230)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:27807/umc/y/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:27807/umc/y/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([59822DFF3468BF56:D1D612259A94D2AE]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:27807 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: 127.0.0.1:25761 failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:25761 failed to respond
	at __randomizedtesting.SeedInfo.seed([59822DFF3468BF56:D1D612259A94D2AE]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:25761 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: 127.0.0.1:54581 failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:54581 failed to respond
	at __randomizedtesting.SeedInfo.seed([59822DFF3468BF56:D1D612259A94D2AE]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:54581 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9902 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/init-core-data-001
   [junit4]   2> 968972 T2732 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /umc/y
   [junit4]   2> 968977 T2732 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 968978 T2733 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 969078 T2732 oasc.ZkTestServer.run start zk server on port:11023
   [junit4]   2> 969079 T2732 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 969080 T2732 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 969085 T2740 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5524d422 name:ZooKeeperConnection Watcher:127.0.0.1:11023 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 969085 T2732 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 969086 T2732 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 969086 T2732 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 969089 T2732 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 969090 T2732 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 969091 T2743 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1b7f8ea7 name:ZooKeeperConnection Watcher:127.0.0.1:11023/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 969091 T2732 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 969091 T2732 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 969092 T2732 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 969093 T2732 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 969095 T2732 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 969096 T2732 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 969097 T2732 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 969098 T2732 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 969100 T2732 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 969101 T2732 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 969204 T2732 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 969205 T2732 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 969206 T2732 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 969207 T2732 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 969208 T2732 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 969209 T2732 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 969210 T2732 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 969211 T2732 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 969213 T2732 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 969214 T2732 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 969215 T2732 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 969216 T2732 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 969218 T2732 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 969218 T2732 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 969220 T2732 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 969221 T2732 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 969222 T2732 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 969223 T2732 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 970387 T2732 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/control-001/cores/collection1
   [junit4]   2> 970390 T2732 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 970392 T2732 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:27802
   [junit4]   2> 970393 T2732 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/control-001/cores, hostPort=27801, hostContext=/umc/y}
   [junit4]   2> 970393 T2732 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 970393 T2732 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/control-001/'
   [junit4]   2> 970422 T2732 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/control-001/solr.xml
   [junit4]   2> 970457 T2732 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/control-001/cores
   [junit4]   2> 970458 T2732 oasc.CoreContainer.<init> New CoreContainer 1882061578
   [junit4]   2> 970458 T2732 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/control-001/]
   [junit4]   2> 970458 T2732 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/control-001/lib
   [junit4]   2> 970459 T2732 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/control-001/lib).
   [junit4]   2> 970468 T2732 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 970468 T2732 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 970468 T2732 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 970468 T2732 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 970469 T2732 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 970469 T2732 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 970469 T2732 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 970469 T2732 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 970469 T2732 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 970470 T2732 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 970470 T2732 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 970474 T2732 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 970475 T2732 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 970475 T2732 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 970476 T2732 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 970476 T2732 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:11023/solr
   [junit4]   2> 970476 T2732 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 970476 T2732 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 970477 T2732 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 970478 T2755 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@40b21de1 name:ZooKeeperConnection Watcher:127.0.0.1:11023 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 970479 T2732 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 970479 T2732 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 970480 T2732 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 970481 T2758 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1df671fd name:ZooKeeperConnection Watcher:127.0.0.1:11023/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 970481 T2732 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 970482 T2732 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 970484 T2732 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 970485 T2732 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 970486 T2732 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 970488 T2732 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 970489 T2732 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 970490 T2732 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 970491 T2732 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 970492 T2732 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:27801_umc%2Fy
   [junit4]   2> 970492 T2732 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:27801_umc%2Fy
   [junit4]   2> 970493 T2732 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 970494 T2732 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 970495 T2732 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 970496 T2732 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:27801_umc%2Fy
   [junit4]   2> 970496 T2732 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 970497 T2732 oasc.Overseer.start Overseer (id=93597250280488963-127.0.0.1:27801_umc%2Fy-n_0000000000) starting
   [junit4]   2> 970499 T2732 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 970502 T2732 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 970503 T2760 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 970503 T2732 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 970503 T2759 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 970506 T2732 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/control-001/cores
   [junit4]   2> 970508 T2732 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {shard=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/control-001/cores/collection1/, loadOnStartup=true, name=collection1, config=solrconfig.xml, coreNodeName=, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/control-001/cores/collection1, collection=control_collection, schema=schema.xml, dataDir=data/}
   [junit4]   2> 970508 T2732 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/control-001/cores/collection1/
   [junit4]   2> 970508 T2732 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 970509 T2762 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 970509 T2762 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 970510 T2762 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 970510 T2758 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 970511 T2759 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:27801_umc%2Fy",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:27801/umc/y",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null} current state version: 0
   [junit4]   2> 970511 T2759 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:27801_umc%2Fy",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:27801/umc/y",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 970511 T2759 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 970511 T2759 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 970512 T2758 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 971510 T2762 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 971511 T2762 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 971511 T2762 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 971512 T2762 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 971512 T2762 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 971513 T2762 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/control-001/cores/collection1/'
   [junit4]   2> 971531 T2762 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 971533 T2762 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 971546 T2762 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 971583 T2762 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 971638 T2762 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 971740 T2762 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 971751 T2762 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 972189 T2762 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 972208 T2762 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 972211 T2762 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 972221 T2762 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 972226 T2762 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 972230 T2762 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 972231 T2762 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 972231 T2762 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 972232 T2762 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 972232 T2762 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 972233 T2762 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 972233 T2762 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 972233 T2762 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 972233 T2762 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/control-001/cores/collection1/data/
   [junit4]   2> 972234 T2762 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5c2ae567
   [junit4]   2> 972235 T2762 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/control-001/cores/collection1/data
   [junit4]   2> 972235 T2762 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/control-001/cores/collection1/data/index/
   [junit4]   2> 972235 T2762 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 972236 T2762 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/control-001/cores/collection1/data/index
   [junit4]   2> 972236 T2762 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=15, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=15.57421875, floorSegmentMB=0.9921875, forceMergeDeletesPctAllowed=2.274594146939587, segmentsPerTier=28.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 972237 T2762 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@68362e4f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3d1a79fc),segFN=segments_1,generation=1}
   [junit4]   2> 972237 T2762 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 972243 T2762 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 972243 T2762 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 972243 T2762 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 972244 T2762 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 972244 T2762 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 972244 T2762 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 972245 T2762 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 972245 T2762 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 972245 T2762 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 972273 T2762 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 972275 T2762 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 972278 T2762 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 972280 T2762 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 972286 T2762 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /get,/admin/file,/schema,/admin/luke,/admin/plugins,/admin/threads,/admin/segments,/admin/mbeans,/update,/replication,/admin/ping,/update/json,/update/json/docs,/update/csv,/admin/logging,standard,/admin/system,/config,/admin/properties
   [junit4]   2> 972288 T2762 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 972290 T2762 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 972290 T2762 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 972292 T2762 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 972292 T2762 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 972293 T2762 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=29, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=86.19140625, floorSegmentMB=0.2578125, forceMergeDeletesPctAllowed=11.354393203908225, segmentsPerTier=45.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 972293 T2762 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@68362e4f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3d1a79fc),segFN=segments_1,generation=1}
   [junit4]   2> 972294 T2762 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 972294 T2762 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@53c4f39f[collection1] main
   [junit4]   2> 972294 T2762 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 972295 T2762 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 972295 T2762 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 972296 T2762 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 972296 T2762 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 972296 T2762 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 972297 T2762 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 972297 T2762 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 972297 T2762 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 972298 T2762 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 972299 T2763 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@53c4f39f[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 972299 T2762 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 972300 T2766 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:27801/umc/y collection:control_collection shard:shard1
   [junit4]   2> 972300 T2732 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 972301 T2732 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 972301 T2766 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 972304 T2766 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 972306 T2758 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 972306 T2766 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 972306 T2732 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 972306 T2759 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 972306 T2766 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C331 name=collection1 org.apache.solr.core.SolrCore@5cff8658 url=http://127.0.0.1:27801/umc/y/collection1 node=127.0.0.1:27801_umc%2Fy C331_STATE=coll:control_collection core:collection1 props:{state=down, node_name=127.0.0.1:27801_umc%2Fy, core=collection1, base_url=http://127.0.0.1:27801/umc/y}
   [junit4]   2> 972307 T2766 C:control_collection S:shard1 C331 P27801 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:27801/umc/y/collection1/
   [junit4]   2> 972307 T2732 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 972307 T2766 C:control_collection S:shard1 C331 P27801 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 972308 T2766 C:control_collection S:shard1 C331 P27801 oasc.SyncStrategy.syncToMe http://127.0.0.1:27801/umc/y/collection1/ has no replicas
   [junit4]   2> 972308 T2766 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:27801/umc/y/collection1/ shard1
   [junit4]   2> 972308 T2766 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 972308 T2770 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5418b778 name:ZooKeeperConnection Watcher:127.0.0.1:11023/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 972308 T2758 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 972309 T2732 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 972309 T2732 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 972310 T2732 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 972311 T2758 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 972312 T2759 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:27801/umc/y",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 972312 T2732 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 972416 T2770 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 972416 T2758 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 972465 T2766 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:27801/umc/y/collection1/ and leader is http://127.0.0.1:27801/umc/y/collection1/
   [junit4]   2> 972466 T2766 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:27801/umc/y
   [junit4]   2> 972466 T2766 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 972466 T2766 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 972466 T2766 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 972467 T2758 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 972468 T2759 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:27801_umc%2Fy",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:27801/umc/y",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1"} current state version: 3
   [junit4]   2> 972468 T2759 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:27801_umc%2Fy",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:27801/umc/y",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1"}
   [junit4]   2> 972571 T2758 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 972571 T2770 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 973746 T2732 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001/cores/collection1
   [junit4]   2> 973749 T2732 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001
   [junit4]   2> 973749 T2732 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 973752 T2732 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:27808
   [junit4]   2> 973752 T2732 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/tempDir-001/jetty1, hostContext=/umc/y, hostPort=27807, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001/cores, solrconfig=solrconfig.xml}
   [junit4]   2> 973753 T2732 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 973753 T2732 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001/'
   [junit4]   2> 973790 T2732 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001/solr.xml
   [junit4]   2> 973857 T2732 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001/cores
   [junit4]   2> 973857 T2732 oasc.CoreContainer.<init> New CoreContainer 326067208
   [junit4]   2> 973858 T2732 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001/]
   [junit4]   2> 973858 T2732 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001/lib
   [junit4]   2> 973859 T2732 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001/lib).
   [junit4]   2> 973873 T2732 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 973874 T2732 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 973874 T2732 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 973874 T2732 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 973875 T2732 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 973875 T2732 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 973875 T2732 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 973876 T2732 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 973876 T2732 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 973876 T2732 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 973877 T2732 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 973887 T2732 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 973888 T2732 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 973888 T2732 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 973889 T2732 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 973889 T2732 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:11023/solr
   [junit4]   2> 973889 T2732 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 973890 T2732 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 973891 T2732 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 973892 T2782 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@64dccefe name:ZooKeeperConnection Watcher:127.0.0.1:11023 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 973893 T2732 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 973894 T2732 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 973896 T2732 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 973897 T2785 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3ea6b446 name:ZooKeeperConnection Watcher:127.0.0.1:11023/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 973897 T2732 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 973901 T2732 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 974906 T2732 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:27807_umc%2Fy
   [junit4]   2> 974907 T2732 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:27807_umc%2Fy
   [junit4]   2> 974910 T2732 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 974912 T2732 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001/cores
   [junit4]   2> 974914 T2732 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001/cores/collection1, config=solrconfig.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001/cores/collection1/, transient=false, loadOnStartup=true, coreNodeName=, collection=collection1, shard=, name=collection1, dataDir=data/, schema=schema.xml}
   [junit4]   2> 974914 T2732 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001/cores/collection1/
   [junit4]   2> 974914 T2732 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 974915 T2786 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 974915 T2786 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 974916 T2758 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 974916 T2786 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 974917 T2759 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:27807_umc%2Fy",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:27807/umc/y",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null} current state version: 4
   [junit4]   2> 974917 T2759 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:27807_umc%2Fy",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:27807/umc/y",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 974917 T2759 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 974918 T2759 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 974919 T2770 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 974919 T2785 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 974919 T2758 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 975916 T2786 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 975917 T2786 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 975917 T2786 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 975918 T2786 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 975918 T2786 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 975918 T2786 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001/cores/collection1/'
   [junit4]   2> 975931 T2786 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 975933 T2786 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 975940 T2786 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 975956 T2786 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 975989 T2786 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 976091 T2786 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 976098 T2786 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 976387 T2786 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 976401 T2786 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 976403 T2786 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 976410 T2786 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 976414 T2786 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 976417 T2786 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 976417 T2786 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 976417 T2786 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 976418 T2786 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 976418 T2786 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 976418 T2786 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 976418 T2786 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 976419 T2786 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 976419 T2786 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 976419 T2786 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5c2ae567
   [junit4]   2> 976420 T2786 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001/cores/collection1/data
   [junit4]   2> 976420 T2786 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 976420 T2786 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 976420 T2786 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 59822DFF3468BF56-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 976421 T2786 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=15, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=15.57421875, floorSegmentMB=0.9921875, forceMergeDeletesPctAllowed=2.274594146939587, segmentsPerTier=28.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 976421 T2786 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2ba8d1c0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5862aaa4),segFN=segments_1,generation=1}
   [junit4]   2> 976421 T2786 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 976426 T2786 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 976426 T2786 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 976427 T2786 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 976427 T2786 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 976427 T2786 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 976427 T2786 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 976428 T2786 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 976428 T2786 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 976428 T2786 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 976450 T2786 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 976452 T2786 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 976454 T2786 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 976456 T2786 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 976461 T2786 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /get,/admin/file,/schema,/admin/luke,/admin/plugins,/admin/threads,/admin/segments,/admin/mbeans,/update,/replication,/admin/ping,/update/json,/update/json/docs,/update/csv,/admin/logging,standard,/admin/system,/config,/admin/properties
   [junit4]   2> 976463 T2786 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 976464 T2786 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 976464 T2786 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 976466 T2786 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 976466 T2786 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 976467 T2786 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=29, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=86.19140625, floorSegmentMB=0.2578125, forceMergeDeletesPctAllowed=11.354393203908225, segmentsPerTier=45.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 976468 T2786 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2ba8d1c0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5862aaa4),segFN=segments_1,generation=1}
   [junit4]   2> 976468 T2786 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 976468 T2786 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@6fe92c91[collection1] main
   [junit4]   2> 976469 T2786 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 976469 T2786 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 976470 T2786 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 976470 T2786 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 976470 T2786 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 976470 T2786 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 976471 T2786 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 976471 T2786 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 976471 T2786 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 976471 T2786 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 976472 T2787 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6fe92c91[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 976473 T2786 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 976473 T2790 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:27807/umc/y collection:collection1 shard:shard1
   [junit4]   2> 976474 T2732 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 976474 T2790 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 976474 T2732 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 976477 T2790 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 976478 T2758 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 976478 T2790 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 976478 T2790 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C332 name=collection1 org.apache.solr.core.SolrCore@1bc9848f url=http://127.0.0.1:27807/umc/y/collection1 node=127.0.0.1:27807_umc%2Fy C332_STATE=coll:collection1 core:collection1 props:{state=down, node_name=127.0.0.1:27807_umc%2Fy, core=collection1, base_url=http://127.0.0.1:27807/umc/y}
   [junit4]   2> 976478 T2790 C:collection1 S:shard1 C332 P27807 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:27807/umc/y/collection1/
   [junit4]   2> 976479 T2759 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 976479 T2790 C:collection1 S:shard1 C332 P27807 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 976479 T2790 C:collection1 S:shard1 C332 P27807 oasc.SyncStrategy.syncToMe http://127.0.0.1:27807/umc/y/collection1/ has no rep

[...truncated too long message...]

y: stop shard! 29705
   [junit4]   2> 1582636 T3248 C:control_collection S:shard1 c:collection1 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=192973817
   [junit4]   2> 1582636 T3248 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1582636 T3248 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1582638 T3409 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1582639 T3248 C:control_collection S:shard1 c:collection1 oasc.Overseer.close Overseer (id=93597279702548490-127.0.0.1:29705_qiue%2Fi-n_0000000003) closing
   [junit4]   2> 1582639 T3411 oasc.Overseer$ClusterStateUpdater.run Overseer Loop exiting : 127.0.0.1:29705_qiue%2Fi
   [junit4]   2> 1582641 T3409 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 1583445 T3402 C457 P29705 oasc.SyncStrategy.sync WARN Closed, skipping sync up.
   [junit4]   2> 1583447 T3402 oasc.ShardLeaderElectionContext.rejoinLeaderElection Not rejoining election because CoreContainer is closed
   [junit4]   2> 1583447 T3402 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@1957e1b4
   [junit4]   2> 1583447 T3402 oasc.ZkController.unRegisterConfListener  a listener was removed because of core close
   [junit4]   2> 1583448 T3402 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=2,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=501,cumulative_deletesById=241,cumulative_deletesByQuery=1,cumulative_errors=0,transaction_logs_total_size=73417,transaction_logs_total_number=1}
   [junit4]   2> 1583449 T3402 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 1583450 T3402 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 1583450 T3402 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 1583452 T3402 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 1583471 T3402 oasc.CachingDirectoryFactory.close Closing StandardDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 1583471 T3402 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 59822DFF3468BF56-001/shard-3-001/cores/collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 59822DFF3468BF56-001/shard-3-001/cores/collection1/data;done=false>>]
   [junit4]   2> 1583471 T3402 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 59822DFF3468BF56-001/shard-3-001/cores/collection1/data
   [junit4]   2> 1583472 T3402 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 59822DFF3468BF56-001/shard-3-001/cores/collection1/data/index [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 59822DFF3468BF56-001/shard-3-001/cores/collection1/data/index;done=false>>]
   [junit4]   2> 1583472 T3402 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 59822DFF3468BF56-001/shard-3-001/cores/collection1/data/index
   [junit4]   2> 1583474 T3248 C:control_collection S:shard1 c:collection1 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/qiue/i,null}
   [junit4]   2> 1583677 T3248 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:26120 26120
   [junit4]   2> 1583737 T3377 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:26120 26120
   [junit4]   2> 1583741 T3377 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		2	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ChaosMonkeyNothingIsSafeTest -Dtests.method=test -Dtests.seed=59822DFF3468BF56 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=sv_SE -Dtests.timezone=America/Argentina/Buenos_Aires -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE  166s J1 | ChaosMonkeyNothingIsSafeTest.test <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: There were too many update fails (31 > 20) - we expect it can happen, but shouldn't easily
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([59822DFF3468BF56:D1D612259A94D2AE]:0)
   [junit4]    > 	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:230)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1583750 T3248 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest 59822DFF3468BF56-001
   [junit4]   2> 165957 T3247 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {a_i=PostingsFormat(name=Direct), _version_=FST50, id=PostingsFormat(name=Direct), a_t=FST50, rnd_b=PostingsFormat(name=Direct)}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=true,coord=crazy): {}, locale=sv_SE, timezone=America/Argentina/Buenos_Aires
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=75621816,total=241172480
   [junit4]   2> NOTE: All tests run in this JVM: [TestLMDirichletSimilarityFactory, CoreMergeIndexesAdminHandlerTest, SolrTestCaseJ4Test, HdfsWriteToMultipleCollectionsTest, TestExpandComponent, TestFieldTypeCollectionResource, DistributedFacetPivotSmallAdvancedTest, TestRTGBase, TestSolrIndexConfig, TestWriterPerf, SimplePostToolTest, TestLuceneMatchVersion, TestCoreDiscovery, TestLFUCache, HdfsLockFactoryTest, TestSimpleTrackingShardHandler, TestReversedWildcardFilterFactory, RequestHandlersTest, CacheHeaderTest, DeleteShardTest, TestLMJelinekMercerSimilarityFactory, AtomicUpdatesTest, SolrXmlInZkTest, OverseerTest, NumericFieldsTest, CollectionsAPIAsyncDistributedZkTest, RollingRestartTest, ClusterStateTest, TestCursorMarkWithoutUniqueKey, TestShortCircuitedRequests, TestBadConfig, SolrCmdDistributorTest, BadCopyFieldTest, TestAnalyzeInfixSuggestions, FacetPivotSmallTest, LeaderFailoverAfterPartitionTest, TestFastWriter, OverseerRolesTest, TestCSVResponseWriter, TestRecoveryHdfs, AsyncMigrateRouteKeyTest, TestCloudManagedSchema, ClusterStateUpdateTest, TestStandardQParsers, HdfsCollectionsAPIDistributedZkTest, SolrIndexConfigTest, ShardSplitTest, TestSolrCoreProperties, CircularListTest, TestCSVLoader, TestManagedSchema, DistributedFacetPivotWhiteBoxTest, PreAnalyzedUpdateProcessorTest, TestMiniSolrCloudClusterSSL, RecoveryAfterSoftCommitTest, TestFaceting, ChaosMonkeyNothingIsSafeTest]
   [junit4] Completed on J1 in 166.51s, 1 test, 1 failure <<< FAILURES!

[...truncated 702 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 1 failure, 61 ignored (21 assumptions)

Total time: 54 minutes 51 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 210106413 bytes
Compression is 0.0%
Took 32 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2890 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2890/

4 tests failed.
REGRESSION:  org.apache.solr.cloud.TestCloudPivotFacet.test

Error Message:
{main(facet=true&facet.pivot=%7B%21stats%3Dst3%7Dpivot_ti%2Cpivot_tl%2Cpivot_dt&facet.limit=16&facet.offset=3&facet.pivot.mincount=3&facet.missing=true&facet.sort=index&facet.overrequest.ratio=-1.1108205),extra(rows=0&q=*%3A*&fq=id%3A%5B*+TO+665%5D&stats=true&stats.field=%7B%21key%3Dsk1+tag%3Dst1%2Cst2%7Dpivot_tdt1&stats.field=%7B%21key%3Dsk2+tag%3Dst2%2Cst3%7Dpivot_ti&stats.field=%7B%21key%3Dsk3+tag%3Dst3%2Cst4%7Dpivot_tl1&_test_min=3&_test_miss=true&_test_sort=index)} ==> pivot_ti,pivot_tl,pivot_dt: {params(rows=0),defaults({main({main(rows=0&q=*%3A*&fq=id%3A%5B*+TO+665%5D&stats=true&stats.field=%7B%21key%3Dsk1+tag%3Dst1%2Cst2%7Dpivot_tdt1&stats.field=%7B%21key%3Dsk2+tag%3Dst2%2Cst3%7Dpivot_ti&stats.field=%7B%21key%3Dsk3+tag%3Dst3%2Cst4%7Dpivot_tl1&_test_min=3&_test_miss=true&_test_sort=index),extra(fq=%7B%21term+f%3Dpivot_ti%7D35)}),extra(fq=-pivot_tl%3A%5B*+TO+*%5D)})} expected:<4> but was:<5>

Stack Trace:
java.lang.AssertionError: {main(facet=true&facet.pivot=%7B%21stats%3Dst3%7Dpivot_ti%2Cpivot_tl%2Cpivot_dt&facet.limit=16&facet.offset=3&facet.pivot.mincount=3&facet.missing=true&facet.sort=index&facet.overrequest.ratio=-1.1108205),extra(rows=0&q=*%3A*&fq=id%3A%5B*+TO+665%5D&stats=true&stats.field=%7B%21key%3Dsk1+tag%3Dst1%2Cst2%7Dpivot_tdt1&stats.field=%7B%21key%3Dsk2+tag%3Dst2%2Cst3%7Dpivot_ti&stats.field=%7B%21key%3Dsk3+tag%3Dst3%2Cst4%7Dpivot_tl1&_test_min=3&_test_miss=true&_test_sort=index)} ==> pivot_ti,pivot_tl,pivot_dt: {params(rows=0),defaults({main({main(rows=0&q=*%3A*&fq=id%3A%5B*+TO+665%5D&stats=true&stats.field=%7B%21key%3Dsk1+tag%3Dst1%2Cst2%7Dpivot_tdt1&stats.field=%7B%21key%3Dsk2+tag%3Dst2%2Cst3%7Dpivot_ti&stats.field=%7B%21key%3Dsk3+tag%3Dst3%2Cst4%7Dpivot_tl1&_test_min=3&_test_miss=true&_test_sort=index),extra(fq=%7B%21term+f%3Dpivot_ti%7D35)}),extra(fq=-pivot_tl%3A%5B*+TO+*%5D)})} expected:<4> but was:<5>
	at __randomizedtesting.SeedInfo.seed([12A3AA716553D370:9AF795ABCBAFBE88]:0)
	at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotCountsAreCorrect(TestCloudPivotFacet.java:281)
	at org.apache.solr.cloud.TestCloudPivotFacet.test(TestCloudPivotFacet.java:228)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.AssertionError: pivot_ti,pivot_tl,pivot_dt: {params(rows=0),defaults({main({main(rows=0&q=*%3A*&fq=id%3A%5B*+TO+665%5D&stats=true&stats.field=%7B%21key%3Dsk1+tag%3Dst1%2Cst2%7Dpivot_tdt1&stats.field=%7B%21key%3Dsk2+tag%3Dst2%2Cst3%7Dpivot_ti&stats.field=%7B%21key%3Dsk3+tag%3Dst3%2Cst4%7Dpivot_tl1&_test_min=3&_test_miss=true&_test_sort=index),extra(fq=%7B%21term+f%3Dpivot_ti%7D35)}),extra(fq=-pivot_tl%3A%5B*+TO+*%5D)})} expected:<4> but was:<5>
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.apache.solr.cloud.TestCloudPivotFacet.assertNumFound(TestCloudPivotFacet.java:677)
	at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotData(TestCloudPivotFacet.java:335)
	at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotCountsAreCorrect(TestCloudPivotFacet.java:302)
	at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotCountsAreCorrect(TestCloudPivotFacet.java:313)
	at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotCountsAreCorrect(TestCloudPivotFacet.java:271)
	... 42 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:13551/lv_/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:13551/lv_/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([12A3AA716553D370:9AF795ABCBAFBE88]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:13551 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:13793/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:13793/collection1
	at __randomizedtesting.SeedInfo.seed([12A3AA716553D370:9AF795ABCBAFBE88]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:13793/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:13793 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:58620/repfacttest_c8n_1x3_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:58620/repfacttest_c8n_1x3_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([12A3AA716553D370:9AF795ABCBAFBE88]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:58620/repfacttest_c8n_1x3_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:58620 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9791 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/init-core-data-001
   [junit4]   2> 669602 T2624 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 669607 T2624 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 669607 T2625 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 669707 T2624 oasc.ZkTestServer.run start zk server on port:58617
   [junit4]   2> 669708 T2624 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 669709 T2624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 669712 T2632 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7172a99e name:ZooKeeperConnection Watcher:127.0.0.1:58617 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 669712 T2624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 669713 T2624 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 669713 T2624 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 669715 T2624 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 669716 T2624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 669717 T2635 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4575d7f2 name:ZooKeeperConnection Watcher:127.0.0.1:58617/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 669717 T2624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 669718 T2624 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 669718 T2624 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 669720 T2624 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 669721 T2624 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 669722 T2624 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 669724 T2624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 669724 T2624 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 669726 T2624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 669726 T2624 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 669829 T2624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 669829 T2624 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 669831 T2624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 669831 T2624 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 669832 T2624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 669833 T2624 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 669834 T2624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 669834 T2624 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 669836 T2624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 669836 T2624 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 669837 T2624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 669838 T2624 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 669839 T2624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 669839 T2624 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 669841 T2624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 669841 T2624 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 669842 T2624 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 669843 T2624 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 670822 T2624 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/control-001/cores/collection1
   [junit4]   2> 670826 T2624 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 670828 T2624 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:58621
   [junit4]   2> 670829 T2624 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/tempDir-001/control/data, hostPort=58620, hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/control-001/cores}
   [junit4]   2> 670829 T2624 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 670830 T2624 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/control-001/'
   [junit4]   2> 670863 T2624 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/control-001/solr.xml
   [junit4]   2> 670905 T2624 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/control-001/cores
   [junit4]   2> 670905 T2624 oasc.CoreContainer.<init> New CoreContainer 54629337
   [junit4]   2> 670905 T2624 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/control-001/]
   [junit4]   2> 670906 T2624 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/control-001/lib
   [junit4]   2> 670906 T2624 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/control-001/lib).
   [junit4]   2> 670916 T2624 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 670917 T2624 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 670917 T2624 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 670917 T2624 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 670917 T2624 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 670917 T2624 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 670918 T2624 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 670918 T2624 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 670918 T2624 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 670918 T2624 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 670919 T2624 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 670924 T2624 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 670924 T2624 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 670925 T2624 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 670925 T2624 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 670925 T2624 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:58617/solr
   [junit4]   2> 670925 T2624 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 670926 T2624 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 670926 T2624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 670928 T2647 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4b038a52 name:ZooKeeperConnection Watcher:127.0.0.1:58617 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 670929 T2624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 670929 T2624 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 670931 T2624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 670933 T2650 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@aa6d68c name:ZooKeeperConnection Watcher:127.0.0.1:58617/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 670933 T2624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 670935 T2624 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 670937 T2624 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 670938 T2624 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 670940 T2624 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 670942 T2624 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 670943 T2624 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 670945 T2624 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 670946 T2624 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 670947 T2624 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:58620_
   [junit4]   2> 670947 T2624 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:58620_
   [junit4]   2> 670948 T2624 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 670950 T2624 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 670951 T2624 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 670952 T2624 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:58620_
   [junit4]   2> 670952 T2624 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 670953 T2624 oasc.Overseer.start Overseer (id=93596374343286787-127.0.0.1:58620_-n_0000000000) starting
   [junit4]   2> 670955 T2624 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 670960 T2624 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 670960 T2652 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 670960 T2624 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 670961 T2651 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 670965 T2624 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/control-001/cores
   [junit4]   2> 670967 T2624 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=control_collection, loadOnStartup=true, shard=, schema=schema.xml, name=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/control-001/cores/collection1/, dataDir=data/, coreNodeName=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/control-001/cores/collection1, config=solrconfig.xml, transient=false}
   [junit4]   2> 670967 T2624 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/control-001/cores/collection1/
   [junit4]   2> 670967 T2624 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 670968 T2654 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 670968 T2654 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 670969 T2654 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 670969 T2650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 670970 T2651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:58620_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58620",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 0
   [junit4]   2> 670971 T2651 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:58620_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58620",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection"}
   [junit4]   2> 670971 T2651 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 670971 T2651 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 670972 T2650 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 671970 T2654 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 671971 T2654 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 671971 T2654 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 671971 T2654 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 671972 T2654 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 671972 T2654 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/control-001/cores/collection1/'
   [junit4]   2> 671986 T2654 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 671988 T2654 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 671997 T2654 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 672015 T2654 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 672066 T2654 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 672167 T2654 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 672175 T2654 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 672491 T2654 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 672504 T2654 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 672506 T2654 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 672514 T2654 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 672518 T2654 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 672521 T2654 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 672522 T2654 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 672522 T2654 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 672522 T2654 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 672523 T2654 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 672523 T2654 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 672523 T2654 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 672523 T2654 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 672523 T2654 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/control-001/cores/collection1/data/
   [junit4]   2> 672524 T2654 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3e6cd229
   [junit4]   2> 672524 T2654 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/control-001/cores/collection1/data
   [junit4]   2> 672525 T2654 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/control-001/cores/collection1/data/index/
   [junit4]   2> 672525 T2654 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 672525 T2654 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/control-001/cores/collection1/data/index
   [junit4]   2> 672525 T2654 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=42, maxMergeAtOnceExplicit=46, maxMergedSegmentMB=63.03515625, floorSegmentMB=1.435546875, forceMergeDeletesPctAllowed=5.447954420730018, segmentsPerTier=23.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 672526 T2654 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3c4b7168 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7d0281c4),segFN=segments_1,generation=1}
   [junit4]   2> 672526 T2654 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 672530 T2654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 672530 T2654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 672531 T2654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 672531 T2654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 672531 T2654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 672531 T2654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 672532 T2654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 672532 T2654 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 672532 T2654 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 672549 T2654 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 672552 T2654 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 672553 T2654 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 672555 T2654 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 672559 T2654 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/logging,/replication,/schema,/admin/segments,/admin/luke,/update/json/docs,/get,/admin/file,/update/json,/admin/properties,/admin/system,standard,/admin/plugins,/admin/mbeans,/admin/ping,/admin/threads,/update,/config,/update/csv
   [junit4]   2> 672561 T2654 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 672562 T2654 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 672562 T2654 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 672564 T2654 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 672564 T2654 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 672565 T2654 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=17, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=39.8115234375, floorSegmentMB=0.7158203125, forceMergeDeletesPctAllowed=2.9634851000593088, segmentsPerTier=50.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 672565 T2654 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3c4b7168 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7d0281c4),segFN=segments_1,generation=1}
   [junit4]   2> 672565 T2654 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 672566 T2654 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@77cf9b72[collection1] main
   [junit4]   2> 672566 T2654 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 672567 T2654 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 672567 T2654 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 672567 T2654 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 672567 T2654 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 672567 T2654 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 672568 T2654 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 672568 T2654 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 672568 T2654 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 672568 T2654 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 672569 T2655 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@77cf9b72[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 672570 T2654 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 672570 T2658 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:58620 collection:control_collection shard:shard1
   [junit4]   2> 672570 T2624 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 672571 T2624 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 672571 T2658 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 672574 T2658 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 672574 T2624 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 672575 T2624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 672575 T2650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 672576 T2658 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 672576 T2662 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@49b0d9d1 name:ZooKeeperConnection Watcher:127.0.0.1:58617/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 672576 T2651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 672576 T2658 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 672576 T2624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> ASYNC  NEW_CORE C361 name=collection1 org.apache.solr.core.SolrCore@801aa0a url=http://127.0.0.1:58620/collection1 node=127.0.0.1:58620_ C361_STATE=coll:control_collection core:collection1 props:{state=down, node_name=127.0.0.1:58620_, core=collection1, base_url=http://127.0.0.1:58620}
   [junit4]   2> 672577 T2658 C:control_collection S:shard1 C361 P58620 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:58620/collection1/
   [junit4]   2> 672577 T2624 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 672577 T2658 C:control_collection S:shard1 C361 P58620 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 672577 T2650 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 672577 T2624 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 672577 T2658 C:control_collection S:shard1 C361 P58620 oasc.SyncStrategy.syncToMe http://127.0.0.1:58620/collection1/ has no replicas
   [junit4]   2> 672579 T2658 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:58620/collection1/ shard1
   [junit4]   2> 672579 T2658 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 672581 T2624 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 672581 T2624 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 672581 T2624 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 672582 T2650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 672582 T2624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 672583 T2651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58620",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 672583 T2665 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@38586f43 name:ZooKeeperConnection Watcher:127.0.0.1:58617/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 672583 T2624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 672584 T2624 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 672586 T2650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 672587 T2651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 672587 T2651 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 672588 T2650 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 672588 T2662 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 672633 T2658 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:58620/collection1/ and leader is http://127.0.0.1:58620/collection1/
   [junit4]   2> 672634 T2658 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:58620
   [junit4]   2> 672634 T2658 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 672634 T2658 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 672634 T2658 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 672635 T2650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 672636 T2651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:58620_",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58620",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 3
   [junit4]   2> 672637 T2651 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:58620_",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58620",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection"}
   [junit4]   2> 672637 T2651 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 672638 T2650 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 672638 T2662 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 672742 T2662 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 672742 T2650 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 673545 T2624 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001/cores/collection1
   [junit4]   2> 673547 T2624 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001
   [junit4]   2> 673547 T2624 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 673550 T2624 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:58631
   [junit4]   2> 673550 T2624 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solrconfig=solrconfig.xml, hostPort=58630, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/tempDir-001/jetty1, hostContext=/}
   [junit4]   2> 673551 T2624 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 673551 T2624 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001/'
   [junit4]   2> 673581 T2624 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001/solr.xml
   [junit4]   2> 673616 T2624 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001/cores
   [junit4]   2> 673617 T2624 oasc.CoreContainer.<init> New CoreContainer 592846894
   [junit4]   2> 673617 T2624 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001/]
   [junit4]   2> 673617 T2624 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001/lib
   [junit4]   2> 673618 T2624 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001/lib).
   [junit4]   2> 673627 T2624 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 673627 T2624 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 673628 T2624 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 673628 T2624 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 673628 T2624 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 673628 T2624 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 673628 T2624 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 673629 T2624 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 673629 T2624 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 673629 T2624 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 673629 T2624 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 673634 T2624 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 673635 T2624 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 673635 T2624 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 673635 T2624 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 673635 T2624 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:58617/solr
   [junit4]   2> 673636 T2624 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 673636 T2624 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 673636 T2624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 673638 T2677 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@62a364f name:ZooKeeperConnection Watcher:127.0.0.1:58617 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 673639 T2624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 673639 T2624 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 673641 T2624 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 673643 T2680 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@74e7039a name:ZooKeeperConnection Watcher:127.0.0.1:58617/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 673643 T2624 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 673646 T2624 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 674651 T2624 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:58630_
   [junit4]   2> 674652 T2624 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:58630_
   [junit4]   2> 674655 T2624 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 674657 T2624 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001/cores
   [junit4]   2> 674659 T2624 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {shard=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001/cores/collection1, transient=false, config=solrconfig.xml, schema=schema.xml, collection=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001/cores/collection1/, loadOnStartup=true, name=collection1, coreNodeName=, dataDir=data/}
   [junit4]   2> 674660 T2624 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001/cores/collection1/
   [junit4]   2> 674660 T2624 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 674662 T2681 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 674662 T2681 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 674663 T2650 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 674664 T2681 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 674664 T2681 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 674664 T2651 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:58630_",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58630",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 674664 T2681 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 674665 T2651 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:58630_",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58630",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1"}
   [junit4]   2> 674665 T2651 oasco.ReplicaMutator.updateState Collection already exists with numShards=3
   [junit4]   2> 674665 T2651 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard3
   [junit4]   2> 674665 T2681 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 674665 T2681 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 674666 T2651 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 674667 T2680 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 674667 T2680 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 675666 T2681 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 675667 T2681 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 675667 T2681 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 675668 T2681 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 675668 T2681 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 675668 T2681 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001/cores/collection1/'
   [junit4]   2> 675681 T2681 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 675683 T2681 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 675691 T2681 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 675707 T2681 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 675742 T2681 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 675843 T2681 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 675850 T2681 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 676167 T2681 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 676181 T2681 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 676183 T2681 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 676191 T2681 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 676195 T2681 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 676198 T2681 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 676199 T2681 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 676199 T2681 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 676199 T2681 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 676200 T2681 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 676200 T2681 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 676200 T2681 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 676200 T2681 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 676201 T2681 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 676201 T2681 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3e6cd229
   [junit4]   2> 676201 T2681 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001/cores/collection1/data
   [junit4]   2> 676202 T2681 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 676202 T2681 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 676202 T2681 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12A3AA716553D370-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 676203 T2681 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=42, maxMergeAtOnceExplicit=46, maxMergedSegmentMB=63.03515625, floorSegmentMB=1.435546875, forceMergeDeletesPctAllowed=5.447954420730018, segmentsPerTier=23.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 676203 T2681 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5043cc65 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1320240e),segFN=segments_1,generation=1}
   [junit4]   2> 676203 T2681 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 676208 T2681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 676209 T2681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 676209 T2681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 676209 T2681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 676209 T2681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 676209 T2681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 676210 T2681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 676210 T2681 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 676210 T2681 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 676236 T2681 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 676239 T2681 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 676240 T2681 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 676242 T2681 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 676248 T2681 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/logging,/replication,/schema,/admin/segments,/admin/luke,/update/json/docs,/get,/admin/file,/update/json,/admin/properties,/admin/system,standard,/admin/plugins,/admin/mbeans,/admin/ping,/admin/threads,/update,/config,/update/csv
   [junit4]   2> 676250 T2681 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 676251 T2681 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 676251 T2681 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 676252 T2681 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 676252 T2681 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 676253 T2681 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=17, maxMergeAt

[...truncated too long message...]

;done=false>>]
   [junit4]   2> 1643957 T5295 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet 12A3AA716553D370-001/shard-4-001/cores/collection1/data/index
   [junit4]   2> 1643959 T5131 C:control_collection S:shard1 c:collection1 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/_mlt/rs,null}
   [junit4]   2> 1644163 T5131 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:58148 58148
   [junit4]   2> 1644230 T5132 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:58148 58148
   [junit4]   2> 1644232 T5132 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/aliases.json
   [junit4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 		4	/solr/collections/collection1/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/queue
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestCloudPivotFacet -Dtests.method=test -Dtests.seed=12A3AA716553D370 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=vi -Dtests.timezone=Etc/GMT-11 -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] FAILURE 52.3s J1 | TestCloudPivotFacet.test <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: {main(facet=true&facet.pivot=%7B%21stats%3Dst3%7Dpivot_ti%2Cpivot_tl%2Cpivot_dt&facet.limit=16&facet.offset=3&facet.pivot.mincount=3&facet.missing=true&facet.sort=index&facet.overrequest.ratio=-1.1108205),extra(rows=0&q=*%3A*&fq=id%3A%5B*+TO+665%5D&stats=true&stats.field=%7B%21key%3Dsk1+tag%3Dst1%2Cst2%7Dpivot_tdt1&stats.field=%7B%21key%3Dsk2+tag%3Dst2%2Cst3%7Dpivot_ti&stats.field=%7B%21key%3Dsk3+tag%3Dst3%2Cst4%7Dpivot_tl1&_test_min=3&_test_miss=true&_test_sort=index)} ==> pivot_ti,pivot_tl,pivot_dt: {params(rows=0),defaults({main({main(rows=0&q=*%3A*&fq=id%3A%5B*+TO+665%5D&stats=true&stats.field=%7B%21key%3Dsk1+tag%3Dst1%2Cst2%7Dpivot_tdt1&stats.field=%7B%21key%3Dsk2+tag%3Dst2%2Cst3%7Dpivot_ti&stats.field=%7B%21key%3Dsk3+tag%3Dst3%2Cst4%7Dpivot_tl1&_test_min=3&_test_miss=true&_test_sort=index),extra(fq=%7B%21term+f%3Dpivot_ti%7D35)}),extra(fq=-pivot_tl%3A%5B*+TO+*%5D)})} expected:<4> but was:<5>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([12A3AA716553D370:9AF795ABCBAFBE88]:0)
   [junit4]    > 	at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotCountsAreCorrect(TestCloudPivotFacet.java:281)
   [junit4]    > 	at org.apache.solr.cloud.TestCloudPivotFacet.test(TestCloudPivotFacet.java:228)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: java.lang.AssertionError: pivot_ti,pivot_tl,pivot_dt: {params(rows=0),defaults({main({main(rows=0&q=*%3A*&fq=id%3A%5B*+TO+665%5D&stats=true&stats.field=%7B%21key%3Dsk1+tag%3Dst1%2Cst2%7Dpivot_tdt1&stats.field=%7B%21key%3Dsk2+tag%3Dst2%2Cst3%7Dpivot_ti&stats.field=%7B%21key%3Dsk3+tag%3Dst3%2Cst4%7Dpivot_tl1&_test_min=3&_test_miss=true&_test_sort=index),extra(fq=%7B%21term+f%3Dpivot_ti%7D35)}),extra(fq=-pivot_tl%3A%5B*+TO+*%5D)})} expected:<4> but was:<5>
   [junit4]    > 	at org.apache.solr.cloud.TestCloudPivotFacet.assertNumFound(TestCloudPivotFacet.java:677)
   [junit4]    > 	at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotData(TestCloudPivotFacet.java:335)
   [junit4]    > 	at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotCountsAreCorrect(TestCloudPivotFacet.java:302)
   [junit4]    > 	at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotCountsAreCorrect(TestCloudPivotFacet.java:313)
   [junit4]    > 	at org.apache.solr.cloud.TestCloudPivotFacet.assertPivotCountsAreCorrect(TestCloudPivotFacet.java:271)
   [junit4]    > 	... 42 more
   [junit4]   2> 1644239 T5131 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.TestCloudPivotFacet 12A3AA716553D370-001
   [junit4]   2> 52321 T5130 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {pivot_ti=BlockTreeOrds(blocksize=128), pivot_b=FSTOrd50, pivot_z_s1=BlockTreeOrds(blocksize=128), pivot_tf1=BlockTreeOrds(blocksize=128), pivot_tdt1=PostingsFormat(name=Direct), pivot_tl1=PostingsFormat(name=Direct), pivot_z_s=FSTOrd50, pivot_f1=PostingsFormat(name=Direct), range_facet_l=PostingsFormat(name=Direct), id=Lucene50(blocksize=128), dense_pivot_y_s=BlockTreeOrds(blocksize=128), pivot_y_s1=FSTOrd50, _version_=BlockTreeOrds(blocksize=128), dense_pivot_x_s=Lucene50(blocksize=128), pivot_dt=FSTOrd50, multiDefault=PostingsFormat(name=Direct), dense_pivot_x_s1=PostingsFormat(name=Direct), pivot_x_s=Lucene50(blocksize=128), pivot_y_s=PostingsFormat(name=Direct), pivot_b1=PostingsFormat(name=Direct), pivot_ti1=Lucene50(blocksize=128), pivot_i=PostingsFormat(name=Direct), dense_pivot_ti=BlockTreeOrds(blocksize=128), dense_pivot_i1=FSTOrd50, intDefault=BlockTreeOrds(blocksize=128), dense_pivot_ti1=Lucene50(blocksize=128), pivot_tl=FSTOrd50, pivot_f=FSTOrd50, pivot_d1=BlockTreeOrds(blocksize=128), pivot_tdt=Lucene50(blocksize=128), pivot_x_s1=PostingsFormat(name=Direct), pivot_td1=PostingsFormat(name=Direct), timestamp=PostingsFormat(name=Direct), dense_pivot_y_s1=FSTOrd50, pivot_dt1=PostingsFormat(name=Direct), pivot_tf=Lucene50(blocksize=128), pivot_l1=BlockTreeOrds(blocksize=128), dense_pivot_i=PostingsFormat(name=Direct), pivot_d=Lucene50(blocksize=128), pivot_l=Lucene50(blocksize=128), pivot_td=FSTOrd50, pivot_i1=FSTOrd50}, docValues:{timestamp=DocValuesFormat(name=Lucene50)}, sim=RandomSimilarityProvider(queryNorm=true,coord=crazy): {}, locale=vi, timezone=Etc/GMT-11
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=88106264,total=255852544
   [junit4]   2> NOTE: All tests run in this JVM: [CoreAdminCreateDiscoverTest, TestReplicationHandlerBackup, TestReload, TestCryptoKeys, HdfsChaosMonkeySafeLeaderTest, UUIDUpdateProcessorFallbackTest, StressHdfsTest, ShardSplitTest, SolrInfoMBeanTest, DistributedFacetPivotLongTailTest, TestMaxScoreQueryParser, SpatialRPTFieldTypeTest, TestCursorMarkWithoutUniqueKey, TestManagedResourceStorage, OverseerTest, SystemInfoHandlerTest, TestOmitPositions, PluginInfoTest, SolrXmlInZkTest, DOMUtilTest, BinaryUpdateRequestHandlerTest, TestExpandComponent, BadComponentTest, SolrCoreCheckLockOnStartupTest, DeleteInactiveReplicaTest, CircularListTest, TestQueryUtils, InfoHandlerTest, RemoteQueryErrorTest, TestCloudManagedSchema, TestCloudInspectUtil, ActionThrottleTest, TestElisionMultitermQuery, SuggestComponentTest, TestInitParams, RequiredFieldsTest, DistributedFacetPivotWhiteBoxTest, PingRequestHandlerTest, DocValuesMissingTest, SpellingQueryConverterTest, SuggesterTSTTest, TestRandomFaceting, LeaderFailoverAfterPartitionTest, DirectSolrSpellCheckerTest, TestIBSimilarityFactory, TestJsonFacets, BasicZkTest, TestFieldSortValues, CoreAdminHandlerTest, SampleTest, JSONWriterTest, TestImplicitCoreProperties, TestConfig, CollectionsAPIDistributedZkTest, TestBlobHandler, TestMiniSolrCloudCluster, TriLevelCompositeIdRoutingTest, AssignTest, TestClusterStateMutator, LeaderInitiatedRecoveryOnCommitTest, RecoveryAfterSoftCommitTest, TestCloudPivotFacet]
   [junit4] Completed on J1 in 52.60s, 1 test, 1 failure <<< FAILURES!

[...truncated 685 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 1 failure, 61 ignored (21 assumptions)

Total time: 52 minutes 16 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 231424647 bytes
Compression is 0.0%
Took 43 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2889 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2889/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:26971/pq_dze/w/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:26971/pq_dze/w/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([F2CEA2424A880FB9:7A9A9D98E4746241]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:26971 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: 127.0.0.1:28789 failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:28789 failed to respond
	at __randomizedtesting.SeedInfo.seed([F2CEA2424A880FB9:7A9A9D98E4746241]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:28789 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: 127.0.0.1:59378 failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:59378 failed to respond
	at __randomizedtesting.SeedInfo.seed([F2CEA2424A880FB9:7A9A9D98E4746241]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:59378 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9445 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/init-core-data-001
   [junit4]   2> 226670 T1551 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 226676 T1551 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 226676 T1552 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 226776 T1551 oasc.ZkTestServer.run start zk server on port:28776
   [junit4]   2> 226777 T1551 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 226777 T1551 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 226781 T1559 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@73c8cb12 name:ZooKeeperConnection Watcher:127.0.0.1:28776 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 226781 T1551 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 226781 T1551 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 226782 T1551 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 226784 T1551 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 226784 T1551 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 226786 T1562 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@db0784c name:ZooKeeperConnection Watcher:127.0.0.1:28776/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 226786 T1551 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 226786 T1551 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 226786 T1551 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 226789 T1551 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 226790 T1551 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 226791 T1551 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 226793 T1551 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 226794 T1551 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 226796 T1551 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 226796 T1551 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 226798 T1551 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 226799 T1551 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 226800 T1551 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 226801 T1551 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 226802 T1551 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 226803 T1551 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 226804 T1551 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 226805 T1551 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 226806 T1551 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 226807 T1551 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 226808 T1551 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 226809 T1551 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 226810 T1551 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 226811 T1551 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 226812 T1551 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 226812 T1551 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 226814 T1551 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 226814 T1551 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 228002 T1551 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/control-001/cores/collection1
   [junit4]   2> 228006 T1551 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 228009 T1551 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:28785
   [junit4]   2> 228009 T1551 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, hostPort=28784, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/tempDir-001/control/data}
   [junit4]   2> 228010 T1551 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 228010 T1551 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/control-001/'
   [junit4]   2> 228039 T1551 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/control-001/solr.xml
   [junit4]   2> 228074 T1551 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/control-001/cores
   [junit4]   2> 228075 T1551 oasc.CoreContainer.<init> New CoreContainer 876967474
   [junit4]   2> 228075 T1551 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/control-001/]
   [junit4]   2> 228075 T1551 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/control-001/lib
   [junit4]   2> 228076 T1551 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/control-001/lib).
   [junit4]   2> 228084 T1551 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 228084 T1551 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 228085 T1551 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 228085 T1551 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 228085 T1551 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 228085 T1551 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 228085 T1551 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 228086 T1551 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 228086 T1551 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 228086 T1551 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 228086 T1551 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 228091 T1551 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 228092 T1551 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 228092 T1551 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 228092 T1551 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 228092 T1551 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:28776/solr
   [junit4]   2> 228093 T1551 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 228093 T1551 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 228093 T1551 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 228095 T1574 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3e183ec4 name:ZooKeeperConnection Watcher:127.0.0.1:28776 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 228096 T1551 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 228096 T1551 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 228098 T1551 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 228099 T1577 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1a8f6ee3 name:ZooKeeperConnection Watcher:127.0.0.1:28776/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 228100 T1551 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 228101 T1551 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 228104 T1551 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 228106 T1551 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 228108 T1551 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 228110 T1551 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 228113 T1551 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 228114 T1551 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 228116 T1551 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 228117 T1551 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:28784_
   [junit4]   2> 228118 T1551 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:28784_
   [junit4]   2> 228119 T1551 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 228121 T1551 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 228122 T1551 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 228124 T1551 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:28784_
   [junit4]   2> 228124 T1551 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 228126 T1551 oasc.Overseer.start Overseer (id=93595230515494915-127.0.0.1:28784_-n_0000000000) starting
   [junit4]   2> 228128 T1551 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 228135 T1551 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 228135 T1579 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 228135 T1551 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 228136 T1578 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 228139 T1551 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/control-001/cores
   [junit4]   2> 228141 T1551 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=control_collection, name=collection1, coreNodeName=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/control-001/cores/collection1/, config=solrconfig.xml, dataDir=data/, loadOnStartup=true, transient=false, shard=, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/control-001/cores/collection1}
   [junit4]   2> 228141 T1551 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/control-001/cores/collection1/
   [junit4]   2> 228142 T1551 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 228142 T1581 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 228142 T1581 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 228143 T1577 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 228143 T1581 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 228144 T1578 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28784",
   [junit4]   2> 	  "node_name":"127.0.0.1:28784_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1"} current state version: 0
   [junit4]   2> 228145 T1578 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28784",
   [junit4]   2> 	  "node_name":"127.0.0.1:28784_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 228145 T1578 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 228146 T1578 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 228147 T1577 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 229144 T1581 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 229145 T1581 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 229146 T1581 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 229146 T1581 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 229147 T1581 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 229147 T1581 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/control-001/cores/collection1/'
   [junit4]   2> 229164 T1581 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 229166 T1581 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 229178 T1581 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 229199 T1581 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 229247 T1581 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 229248 T1581 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 229259 T1581 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 229555 T1581 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 229566 T1581 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 229568 T1581 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 229580 T1581 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 229583 T1581 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 229586 T1581 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 229587 T1581 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 229587 T1581 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 229587 T1581 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 229588 T1581 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 229588 T1581 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 229588 T1581 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 229588 T1581 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 229588 T1581 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/control-001/cores/collection1/data/
   [junit4]   2> 229589 T1581 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@59e3c35d
   [junit4]   2> 229589 T1581 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/control-001/cores/collection1/data
   [junit4]   2> 229590 T1581 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/control-001/cores/collection1/data/index/
   [junit4]   2> 229590 T1581 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 229590 T1581 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/control-001/cores/collection1/data/index
   [junit4]   2> 229590 T1581 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=47, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 229591 T1581 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4fcf9353 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@44d15714),segFN=segments_1,generation=1}
   [junit4]   2> 229591 T1581 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 229596 T1581 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 229596 T1581 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 229596 T1581 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 229596 T1581 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 229596 T1581 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 229597 T1581 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 229597 T1581 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 229597 T1581 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 229597 T1581 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 229609 T1581 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 229611 T1581 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 229612 T1581 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 229614 T1581 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 229618 T1581 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/system,/admin/mbeans,standard,/admin/plugins,/replication,/admin/file,/update/json/docs,/admin/logging,/admin/properties,/update,/admin/ping,/admin/threads,/get,/config,/schema,/update/csv,/admin/luke,/update/json,/admin/segments
   [junit4]   2> 229620 T1581 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 229621 T1581 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 229621 T1581 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 229622 T1581 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 229622 T1581 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 229623 T1581 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=47, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.2593310496881438]
   [junit4]   2> 229623 T1581 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4fcf9353 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@44d15714),segFN=segments_1,generation=1}
   [junit4]   2> 229623 T1581 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 229623 T1581 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@367eb371[collection1] main
   [junit4]   2> 229624 T1581 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 229625 T1581 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 229625 T1581 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 229625 T1581 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 229625 T1581 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 229625 T1581 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 229626 T1581 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 229626 T1581 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 229626 T1581 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 229627 T1581 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 229627 T1582 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@367eb371[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 229628 T1581 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 229629 T1585 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:28784 collection:control_collection shard:shard1
   [junit4]   2> 229629 T1551 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 229629 T1551 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 229630 T1585 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 229634 T1585 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 229634 T1551 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 229635 T1551 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 229636 T1577 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 229636 T1585 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 229636 T1589 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6bfdaa4c name:ZooKeeperConnection Watcher:127.0.0.1:28776/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 229636 T1585 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 229637 T1578 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 229637 T1551 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> ASYNC  NEW_CORE C210 name=collection1 org.apache.solr.core.SolrCore@61dab312 url=http://127.0.0.1:28784/collection1 node=127.0.0.1:28784_ C210_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:28784, node_name=127.0.0.1:28784_, state=down, core=collection1}
   [junit4]   2> 229637 T1585 C:control_collection S:shard1 C210 P28784 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:28784/collection1/
   [junit4]   2> 229637 T1551 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 229638 T1585 C:control_collection S:shard1 C210 P28784 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 229638 T1551 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 229638 T1585 C:control_collection S:shard1 C210 P28784 oasc.SyncStrategy.syncToMe http://127.0.0.1:28784/collection1/ has no replicas
   [junit4]   2> 229639 T1585 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:28784/collection1/ shard1
   [junit4]   2> 229639 T1585 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 229641 T1551 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 229643 T1577 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 229643 T1578 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28784",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 1
   [junit4]   2> 229748 T1577 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 229748 T1589 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 229795 T1585 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:28784/collection1/ and leader is http://127.0.0.1:28784/collection1/
   [junit4]   2> 229796 T1585 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:28784
   [junit4]   2> 229796 T1585 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 229796 T1585 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 229796 T1585 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 229798 T1577 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 229799 T1578 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28784",
   [junit4]   2> 	  "node_name":"127.0.0.1:28784_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core":"collection1"} current state version: 2
   [junit4]   2> 229800 T1578 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28784",
   [junit4]   2> 	  "node_name":"127.0.0.1:28784_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 229904 T1589 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 229904 T1577 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 230855 T1551 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001/cores/collection1
   [junit4]   2> 230857 T1551 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001
   [junit4]   2> 230858 T1551 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 230860 T1551 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:28790
   [junit4]   2> 230860 T1551 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/tempDir-001/jetty1, hostPort=28789, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001/cores, solrconfig=solrconfig.xml}
   [junit4]   2> 230861 T1551 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 230861 T1551 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001/'
   [junit4]   2> 230902 T1551 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001/solr.xml
   [junit4]   2> 230953 T1551 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001/cores
   [junit4]   2> 230954 T1551 oasc.CoreContainer.<init> New CoreContainer 420523198
   [junit4]   2> 230954 T1551 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001/]
   [junit4]   2> 230954 T1551 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001/lib
   [junit4]   2> 230955 T1551 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001/lib).
   [junit4]   2> 230964 T1551 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 230965 T1551 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 230965 T1551 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 230965 T1551 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 230965 T1551 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 230965 T1551 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 230966 T1551 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 230966 T1551 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 230966 T1551 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 230966 T1551 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 230967 T1551 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 230973 T1551 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 230974 T1551 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 230974 T1551 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 230975 T1551 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 230975 T1551 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:28776/solr
   [junit4]   2> 230975 T1551 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 230975 T1551 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 230976 T1551 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 230978 T1601 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4d8fdec7 name:ZooKeeperConnection Watcher:127.0.0.1:28776 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 230978 T1551 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 230979 T1551 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 230980 T1551 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 230982 T1604 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@494d0e42 name:ZooKeeperConnection Watcher:127.0.0.1:28776/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 230982 T1551 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 230987 T1551 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 231991 T1551 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:28789_
   [junit4]   2> 231992 T1551 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:28789_
   [junit4]   2> 231994 T1551 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 231996 T1551 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001/cores
   [junit4]   2> 231998 T1551 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=collection1, transient=false, dataDir=data/, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001/cores/collection1/, schema=schema.xml, shard=, coreNodeName=, name=collection1, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001/cores/collection1, loadOnStartup=true}
   [junit4]   2> 231998 T1551 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001/cores/collection1/
   [junit4]   2> 231998 T1551 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 231999 T1605 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 231999 T1605 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 232000 T1577 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 232000 T1605 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 232001 T1578 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28789",
   [junit4]   2> 	  "node_name":"127.0.0.1:28789_",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1"} current state version: 3
   [junit4]   2> 232001 T1578 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28789",
   [junit4]   2> 	  "node_name":"127.0.0.1:28789_",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 232002 T1578 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 232002 T1578 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 232003 T1577 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 232003 T1604 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 232003 T1589 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 233000 T1605 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 233001 T1605 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 233001 T1605 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 233002 T1605 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 233002 T1605 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 233002 T1605 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001/cores/collection1/'
   [junit4]   2> 233014 T1605 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 233017 T1605 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 233024 T1605 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 233039 T1605 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 233072 T1605 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 233073 T1605 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 233080 T1605 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 233346 T1605 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 233357 T1605 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 233359 T1605 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 233371 T1605 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 233374 T1605 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 233377 T1605 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 233378 T1605 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 233378 T1605 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 233378 T1605 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 233379 T1605 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 233379 T1605 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 233379 T1605 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 233380 T1605 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 233380 T1605 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 233380 T1605 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@59e3c35d
   [junit4]   2> 233381 T1605 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001/cores/collection1/data
   [junit4]   2> 233381 T1605 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 233381 T1605 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 233381 T1605 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 233382 T1605 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=47, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 233382 T1605 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@51965c17 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@667813d9),segFN=segments_1,generation=1}
   [junit4]   2> 233382 T1605 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 233388 T1605 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 233388 T1605 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 233388 T1605 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 233389 T1605 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 233389 T1605 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 233389 T1605 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 233389 T1605 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 233390 T1605 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 233390 T1605 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 233402 T1605 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 233403 T1605 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 233405 T1605 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 233407 T1605 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 233412 T1605 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/system,/admin/mbeans,standard,/admin/plugins,/replication,/admin/file,/update/json/docs,/admin/logging,/admin/properties,/update,/admin/ping,/admin/threads,/get,/config,/schema,/update/csv,/admin/luke,/update/json,/admin/segments
   [junit4]   2> 233413 T1605 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 233414 T1605 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 233415 T1605 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 233416 T1605 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 233416 T1605 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 233417 T1605 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=47, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.2593310496881438]
   [junit4]   2> 233417 T1605 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@51965c17 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@667813d9),segFN=segments_1,generation=1}
   [junit4]   2> 233417 T1605 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 233417 T1605 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@2e46cd5b[collection1] main
   [junit4]   2> 233418 T1605 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 233419 T1605 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 233419 T1605 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 233419 T1605 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 233420 T1605 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 233420 T1605 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 233420 T1605 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 233421 T1605 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 233421 T1605 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 233421 T1605 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 233422 T1606 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2e46cd5b[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 233423 T1605 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 233423 T1609 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:28789 collection:collection1 shard:shard1
   [junit4]   2> 233424 T1551 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 233424 T1551 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 233424 T1609 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 233429 T1609 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 233431 T1577 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 233431 T1609 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 233431 T1609 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C211 name=collection1 org.apache.solr.core.SolrCore@2c12d853 url=http://127.0.0.1:28789/collection1 node=127.0.0.1:28789_ C211_STATE=coll:collection1 core:collection1 props:{base_url=http://127.0.0.1:28789, node_name=127.0.0.1:28789_, state=down, core=collection1}
   [junit4]   2> 233432 T1609 C:collection1 S:shard1 C211 P28789 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:28789/collection1/
   [junit4]   2> 233432 T1578 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 4
   [junit4]   2> 233432 T1609 C:collection1 S:shard1 C211 P28789 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 233432 T1609 C:collection1 S:shard1 C211 P28789 oasc.SyncStrategy.syncToMe http://127.0.0.1:28789/collection1/ has no replicas
   [junit4]   2> 233433 T1609 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:28789/collection1/ shard1
   [junit4]   2> 233433 T1609 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 233437 T1577 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 233438 T1578 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28789",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 4
   [junit4]   2> 233541 T1577 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 233541 T1604 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 233541 T1589 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 233590 T1609 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:28789/collection1/ and leader is http://127.0.0.1:28789/collection1/
   [junit4]   2> 233591 T1609 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:28789
   [junit4]   2> 233591 T1609 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 233591 T1609 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 233591 T1609 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 233592 T1577 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 233593 T1578 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28789",
   [junit4]   2> 	  "node_name":"127.0.0.1:28789_",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core":"collection1"} current state version: 5
   [junit4]   2> 233594 T1578 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28789",
   [junit4]   2> 	  "node_name":"127.0.0.1:28789_",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 233697 T1577 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 233697 T1589 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 233697 T1604 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 234946 T1551 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-2-001/cores/collection1
   [junit4]   2> 234949 T1551 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-2-001
   [junit4]   2> 234949 T1551 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 234951 T1551 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:28798
   [junit4]   2> 234952 T1551 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/shard-2-001/cores, hostPort=28797, solrconfig=solrconfig.xml, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F2CEA2424A880FB9-001/tempDir-001/jetty2, hostContext=/}
   

[...truncated too long message...]

solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:142)
   [junit4]   2> 		... 11 more
   [junit4]   2> 	
   [junit4]   2> 1213402 T2415 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 1213404 T2198 C:control_collection S:shard1 c:collection1 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/pq_dze/w,null}
   [junit4]   2> 1213426 T2198 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:26935 26935
   [junit4]   2> 1213524 T2199 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:26935 26935
   [junit4]   2> 1213525 T2199 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		6	/solr/aliases.json
   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 		3	/solr/collections/c8n_1x3_commits/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/queue
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1213526 T2198 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 11 connections to: http://127.0.0.1:26947/pq_dze/w
   [junit4]   2> 1213526 T2198 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:26941/pq_dze/w
   [junit4]   2> 1213526 T2198 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:26953/pq_dze/w
   [junit4]   2> 1213527 T2198 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:26982/pq_dze/w
   [junit4]   2> 1213527 T2198 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:26971/pq_dze/w
   [junit4]   2> 1213528 T2198 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:26947/pq_dze/w
   [junit4]   2> 1213528 T2198 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:26941/pq_dze/w
   [junit4]   2> 1213528 T2198 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:26953/pq_dze/w
   [junit4]   2> 1213528 T2198 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:26982/pq_dze/w
   [junit4]   2> 1213529 T2198 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:26971/pq_dze/w
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderInitiatedRecoveryOnCommitTest -Dtests.method=test -Dtests.seed=F2CEA2424A880FB9 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=hu_HU -Dtests.timezone=America/Barbados -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   76.7s J2 | LeaderInitiatedRecoveryOnCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:26971/pq_dze/w/c8n_1x3_commits_shard1_replica1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([F2CEA2424A880FB9:7A9A9D98E4746241]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:26971 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 48 more
   [junit4]   2> 1213549 T2198 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F2CEA2424A880FB9-001
   [junit4]   2> 76699 T2197 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=true,coord=crazy): {}, locale=hu_HU, timezone=America/Barbados
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=188234224,total=325582848
   [junit4]   2> NOTE: All tests run in this JVM: [NoCacheHeaderTest, TestManagedSchemaFieldResource, TestRecoveryHdfs, TestDistributedSearch, DocumentAnalysisRequestHandlerTest, TestStandardQParsers, TestReload, DocValuesMissingTest, RegexBoostProcessorTest, TestFunctionQuery, TestStressVersions, TestXIncludeConfig, TestSolrXml, TestInitParams, ChaosMonkeySafeLeaderTest, TestBlendedInfixSuggestions, TestPerFieldSimilarity, RecoveryZkTest, TestBadConfig, TestRebalanceLeaders, SyncSliceTest, AssignTest, SaslZkACLProviderTest, TestHighlightDedupGrouping, TestLMJelinekMercerSimilarityFactory, DistribJoinFromCollectionTest, DistributedMLTComponentTest, MigrateRouteKeyTest, DocumentBuilderTest, TestSchemaVersionResource, OverseerStatusTest, OpenExchangeRatesOrgProviderTest, TestExtendedDismaxParser, LeaderInitiatedRecoveryOnCommitTest]
   [junit4] Completed on J2 in 77.24s, 1 test, 1 error <<< FAILURES!

[...truncated 867 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 54 minutes 41 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 222602195 bytes
Compression is 0.0%
Took 42 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2888 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2888/

4 tests failed.
REGRESSION:  org.apache.solr.cloud.RecoveryZkTest.test

Error Message:
shard1 is not consistent.  Got 781 from https://127.0.0.1:35615/x_qkw/collection1lastClient and got 165 from https://127.0.0.1:35630/x_qkw/collection1

Stack Trace:
java.lang.AssertionError: shard1 is not consistent.  Got 781 from https://127.0.0.1:35615/x_qkw/collection1lastClient and got 165 from https://127.0.0.1:35630/x_qkw/collection1
	at __randomizedtesting.SeedInfo.seed([88D46D6800DE846B:8052B2AE22E993]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.RecoveryZkTest.test(RecoveryZkTest.java:123)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:62554/kz/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:62554/kz/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([88D46D6800DE846B:8052B2AE22E993]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:62554 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:18591/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:18591/collection1
	at __randomizedtesting.SeedInfo.seed([88D46D6800DE846B:8052B2AE22E993]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:18591/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:18591 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:44740/repfacttest_c8n_1x3_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:44740/repfacttest_c8n_1x3_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([88D46D6800DE846B:8052B2AE22E993]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:44740/repfacttest_c8n_1x3_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:44740 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9325 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/init-core-data-001
   [junit4]   2> 35157 T48 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /kz/
   [junit4]   2> 35233 T48 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 35245 T49 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 35647 T48 oasc.ZkTestServer.run start zk server on port:20767
   [junit4]   2> 35833 T48 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 36134 T48 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 36342 T56 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@a590747 name:ZooKeeperConnection Watcher:127.0.0.1:20767 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 36343 T48 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 36351 T48 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 36365 T48 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 36505 T48 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 36507 T48 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 36515 T59 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@12e82c49 name:ZooKeeperConnection Watcher:127.0.0.1:20767/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 36516 T48 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 36517 T48 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 36534 T48 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 36540 T48 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 36545 T48 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 36549 T48 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 36554 T48 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 36587 T48 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 36597 T48 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 36598 T48 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 36603 T48 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 36604 T48 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 36609 T48 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 36610 T48 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 36614 T48 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 36615 T48 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 36619 T48 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 36620 T48 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 36624 T48 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 36625 T48 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 36629 T48 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 36630 T48 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 36633 T48 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 36634 T48 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 36637 T48 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 36638 T48 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 36642 T48 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 36642 T48 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 38352 T48 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/control-001/cores/collection1
   [junit4]   2> 38358 T48 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 38361 T48 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:39301
   [junit4]   2> 38361 T48 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/kz, hostPort=11539, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/control-001/cores}
   [junit4]   2> 38362 T48 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 38363 T48 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/control-001/'
   [junit4]   2> 38419 T48 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/control-001/solr.xml
   [junit4]   2> 38504 T48 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/control-001/cores
   [junit4]   2> 38505 T48 oasc.CoreContainer.<init> New CoreContainer 1362813671
   [junit4]   2> 38505 T48 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/control-001/]
   [junit4]   2> 38505 T48 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/control-001/lib
   [junit4]   2> 38507 T48 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/control-001/lib).
   [junit4]   2> 38521 T48 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 38521 T48 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 38522 T48 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 38522 T48 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 38523 T48 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 38523 T48 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 38523 T48 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 38524 T48 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 38524 T48 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 38525 T48 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 38525 T48 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 38536 T48 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 38537 T48 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 38538 T48 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 38538 T48 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 38538 T48 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:20767/solr
   [junit4]   2> 38618 T48 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 38619 T48 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 38620 T48 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 38622 T71 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@400f1a30 name:ZooKeeperConnection Watcher:127.0.0.1:20767 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 38623 T48 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 38624 T48 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 38662 T48 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 38664 T74 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@9a83d99 name:ZooKeeperConnection Watcher:127.0.0.1:20767/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 38664 T48 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 38695 T48 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 38719 T48 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 38730 T48 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 38735 T48 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 38739 T48 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 38774 T48 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 38778 T48 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 38781 T48 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 38783 T48 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:11539_kz
   [junit4]   2> 38795 T48 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:11539_kz
   [junit4]   2> 38800 T48 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 38804 T48 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 38808 T48 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 38836 T48 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:11539_kz
   [junit4]   2> 38837 T48 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 38839 T48 oasc.Overseer.start Overseer (id=93594278762905603-127.0.0.1:11539_kz-n_0000000000) starting
   [junit4]   2> 38869 T48 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 38908 T48 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 39452 T76 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 39452 T48 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 39493 T75 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 39506 T48 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/control-001/cores
   [junit4]   2> 39533 T48 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {shard=, loadOnStartup=true, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/control-001/cores/collection1/, dataDir=data/, collection=control_collection, schema=schema.xml, name=collection1, coreNodeName=, config=solrconfig.xml, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/control-001/cores/collection1}
   [junit4]   2> 39533 T48 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/control-001/cores/collection1/
   [junit4]   2> 39534 T48 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 39536 T78 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 39537 T78 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 39567 T74 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 39567 T78 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 39579 T75 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11539/kz",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:11539_kz",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null} current state version: 0
   [junit4]   2> 39673 T75 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11539/kz",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:11539_kz",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 39684 T75 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 39777 T75 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 39807 T74 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 40570 T78 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 40570 T78 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 40572 T78 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 40572 T78 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 40574 T78 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 40575 T78 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/control-001/cores/collection1/'
   [junit4]   2> 40645 T78 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 40673 T78 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 40730 T78 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 40787 T78 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 40902 T78 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 40905 T78 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 41064 T78 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 42901 T78 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 43008 T78 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 43018 T78 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 43383 T78 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 43424 T78 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 43431 T78 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 43441 T78 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 43441 T78 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 43442 T78 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 43444 T78 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 43444 T78 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 43464 T78 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 43465 T78 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 43466 T78 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/control-001/cores/collection1/data/
   [junit4]   2> 43481 T78 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5d098870
   [junit4]   2> 43652 T78 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/control-001/cores/collection1/data
   [junit4]   2> 43653 T78 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/control-001/cores/collection1/data/index/
   [junit4]   2> 43653 T78 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 43654 T78 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/control-001/cores/collection1/data/index
   [junit4]   2> 43655 T78 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=25, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=38.2470703125, floorSegmentMB=1.8984375, forceMergeDeletesPctAllowed=28.361484787913188, segmentsPerTier=26.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 43656 T78 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@73dd66b6 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4c827227),segFN=segments_1,generation=1}
   [junit4]   2> 43656 T78 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 43670 T78 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 43851 T78 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 43869 T78 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 43869 T78 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 43870 T78 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 43870 T78 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 44097 T78 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 44098 T78 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 44098 T78 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 44125 T78 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 44138 T78 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 44141 T78 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 44145 T78 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 44202 T78 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update,/admin/threads,/admin/luke,/admin/logging,/admin/ping,/replication,/update/csv,/update/json,/admin/segments,/admin/mbeans,/update/json/docs,/admin/properties,/admin/file,/config,/admin/system,/schema,/admin/plugins,standard,/get
   [junit4]   2> 44211 T78 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 44234 T78 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 44234 T78 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 44256 T78 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 44257 T78 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 44258 T78 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=45, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=65.017578125, floorSegmentMB=1.97265625, forceMergeDeletesPctAllowed=18.72732245389139, segmentsPerTier=46.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8059727721030615
   [junit4]   2> 44260 T78 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@73dd66b6 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4c827227),segFN=segments_1,generation=1}
   [junit4]   2> 44260 T78 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 44261 T78 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@68543a0[collection1] main
   [junit4]   2> 44261 T78 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 44263 T78 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 44264 T78 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 44282 T78 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 44283 T78 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 44284 T78 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 44285 T78 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 44286 T78 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 44287 T78 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 44336 T78 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 44341 T79 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@68543a0[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 44348 T78 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 44349 T82 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:11539/kz collection:control_collection shard:shard1
   [junit4]   2> 44350 T48 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 44351 T48 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 44421 T82 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 44431 T82 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 44435 T74 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 44436 T82 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 44436 T82 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 44437 T75 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C4 name=collection1 org.apache.solr.core.SolrCore@291d335e url=http://127.0.0.1:11539/kz/collection1 node=127.0.0.1:11539_kz C4_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:11539/kz, core=collection1, state=down, node_name=127.0.0.1:11539_kz}
   [junit4]   2> 44437 T82 C:control_collection S:shard1 C4 P11539 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:11539/kz/collection1/
   [junit4]   2> 44439 T82 C:control_collection S:shard1 C4 P11539 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 44439 T82 C:control_collection S:shard1 C4 P11539 oasc.SyncStrategy.syncToMe http://127.0.0.1:11539/kz/collection1/ has no replicas
   [junit4]   2> 44440 T82 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:11539/kz/collection1/ shard1
   [junit4]   2> 44451 T74 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 44456 T48 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 44457 T48 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 44459 T86 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4aa67956 name:ZooKeeperConnection Watcher:127.0.0.1:20767/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 44460 T48 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 44461 T48 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 44462 T48 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 44462 T82 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 44471 T74 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 44483 T48 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 44484 T75 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11539/kz",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 44591 T74 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 44591 T86 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 44636 T82 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:11539/kz/collection1/ and leader is http://127.0.0.1:11539/kz/collection1/
   [junit4]   2> 44645 T82 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:11539/kz
   [junit4]   2> 44645 T82 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 44646 T82 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 44646 T82 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 44648 T74 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 44651 T75 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11539/kz",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:11539_kz",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1"} current state version: 3
   [junit4]   2> 44652 T75 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11539/kz",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:11539_kz",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1"}
   [junit4]   2> 44759 T86 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 44759 T74 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 46370 T48 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001/cores/collection1
   [junit4]   2> 46373 T48 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001
   [junit4]   2> 46374 T48 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 46377 T48 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:19938
   [junit4]   2> 46377 T48 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=40835, hostContext=/kz, solrconfig=solrconfig.xml, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/tempDir-001/jetty1}
   [junit4]   2> 46378 T48 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 46379 T48 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001/'
   [junit4]   2> 46424 T48 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001/solr.xml
   [junit4]   2> 46486 T48 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001/cores
   [junit4]   2> 46487 T48 oasc.CoreContainer.<init> New CoreContainer 1626291339
   [junit4]   2> 46487 T48 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001/]
   [junit4]   2> 46488 T48 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001/lib
   [junit4]   2> 46489 T48 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001/lib).
   [junit4]   2> 46503 T48 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 46503 T48 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 46504 T48 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 46504 T48 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 46505 T48 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 46505 T48 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 46506 T48 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 46506 T48 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 46507 T48 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 46507 T48 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 46508 T48 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 46515 T48 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 46516 T48 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 46517 T48 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 46517 T48 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 46517 T48 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:20767/solr
   [junit4]   2> 46518 T48 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 46518 T48 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 46519 T48 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 46522 T98 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@203ad0b0 name:ZooKeeperConnection Watcher:127.0.0.1:20767 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 46523 T48 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 46524 T48 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 46527 T48 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 46529 T101 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@294d765e name:ZooKeeperConnection Watcher:127.0.0.1:20767/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 46529 T48 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 46539 T48 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 47548 T48 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40835_kz
   [junit4]   2> 47550 T48 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:40835_kz
   [junit4]   2> 47555 T48 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 47569 T48 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001/cores
   [junit4]   2> 47572 T48 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {loadOnStartup=true, coreNodeName=, shard=, schema=schema.xml, transient=false, config=solrconfig.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001/cores/collection1/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001/cores/collection1, dataDir=data/, name=collection1, collection=collection1}
   [junit4]   2> 47573 T48 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001/cores/collection1/
   [junit4]   2> 47574 T48 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 47577 T102 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 47577 T102 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 47579 T74 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 47579 T102 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 47581 T75 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:40835/kz",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:40835_kz",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null} current state version: 4
   [junit4]   2> 47582 T75 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:40835/kz",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:40835_kz",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 47583 T75 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 47584 T75 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 47587 T86 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 47587 T74 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 47587 T101 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 48580 T102 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 48582 T102 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 48583 T102 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 48584 T102 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 48586 T102 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 48586 T102 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001/cores/collection1/'
   [junit4]   2> 48613 T102 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 48618 T102 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 48658 T102 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 48697 T102 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 48830 T102 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 48833 T102 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 48921 T102 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 49384 T102 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 49433 T102 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 49437 T102 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 49474 T102 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 49480 T102 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 49486 T102 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 49488 T102 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 49489 T102 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 49489 T102 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 49491 T102 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 49491 T102 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 49492 T102 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 49492 T102 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 49492 T102 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 49493 T102 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5d098870
   [junit4]   2> 49494 T102 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001/cores/collection1/data
   [junit4]   2> 49495 T102 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 49495 T102 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 49496 T102 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 88D46D6800DE846B-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 49496 T102 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=25, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=38.2470703125, floorSegmentMB=1.8984375, forceMergeDeletesPctAllowed=28.361484787913188, segmentsPerTier=26.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 49497 T102 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1cf0850b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@72aea404),segFN=segments_1,generation=1}
   [junit4]   2> 49498 T102 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 49510 T102 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 49511 T102 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 49512 T102 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 49512 T102 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 49513 T102 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 49513 T102 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 49514 T102 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 49514 T102 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 49514 T102 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 49542 T102 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 49547 T102 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 49550 T102 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 49554 T102 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 49576 T102 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update,/admin/threads,/admin/luke,/admin/logging,/admin/ping,/replication,/update/csv,/update/json,/admin/segments,/admin/mbeans,/update/json/docs,/admin/properties,/admin/file,/config,/admin/system,/schema,/admin/plugins,standard,/get
   [junit4]   2> 49579 T102 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 49581 T102 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 49582 T102 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 49583 T102 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 49584 T102 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 49586 T102 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=45, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=65.017578125, floorSegmentMB=1.97265625, forceMergeDeletesPctAllowed=18.72732245389139, segmentsPerTier=46.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8059727721030615
   [junit4]   2> 49587 T102 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1cf0850b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@72aea404),segFN=segments_1,generation=1}
   [junit4]   2> 49587 T102 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 49588 T102 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@30eb5ea0[collection1] main
   [junit4]   2> 49588 T102 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 49590 T102 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 49590 T102 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 49591 T102 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 49591 T102 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 49592 T102 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 49593 T102 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 49593 T102 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 49594 T102 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 49594 T102 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 49598 T103 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@30eb5ea0[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 49599 T102 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 49601 T106 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:40835/kz collection:collection1 shard:shard1
   [junit4]   2> 49601 T48 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 49602 T48 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 49602 T106 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 49611 T106 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 49614 T74 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 49614 T106 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 49615 T106 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C5 name=collection1 org.apache.solr.core.SolrCore@2f4f13b8 url=http://127.0.0.1:40835/kz/collection1 node=127.0.0.1:40835_kz C5_STATE=coll:collection1 core:collection1 props:{base_url=http://127.0.0.1:40835/kz, core=collection1, state=down, node_name=127.0.0.1:40835_kz}
   [junit4]   2> 49615 T106 C:collection1 S:shard1 C5 P40835 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:40835/kz/collection1/
   [junit4]   2> 49616 T75 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 49616 T106 C:collection1 S:shard1 C5 P40835 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 49617 T106 C:collection1 S:shard1 C5 P40835 oasc.SyncStrategy.syncToMe http://127.0.0.1:40835/kz/collection1/ has no replicas
   [junit4]   2> 49618 T106 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:40835/kz/collection1/ shard1
   [junit4]   2> 49619 T106 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 49620 T101 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 49620 T74 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent stat

[...truncated too long message...]

0.1:44740/
   [junit4]   2> 1484626 T3118 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:44702/
   [junit4]   2> 1484626 T3118 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:44692/
   [junit4]   2> 1484627 T3118 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:44748/
   [junit4]   2> 1484627 T3118 C:control_collection S:shard1 c:collection1 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 1484627 T3118 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:44740/
   [junit4]   2> 1484628 T3118 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:44702/
   [junit4]   2> 1484628 T3118 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:44692/
   [junit4]   2> 1484628 T3118 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:44748/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=88D46D6800DE846B -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ar_LB -Dtests.timezone=Canada/Newfoundland -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   76.7s J1 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:44740/repfacttest_c8n_1x3_shard1_replica3
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([88D46D6800DE846B:8052B2AE22E993]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:44740/repfacttest_c8n_1x3_shard1_replica3
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 48 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:44740 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 53 more
   [junit4]   2> 1484655 T3118 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 88D46D6800DE846B-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {intDefault=PostingsFormat(name=Direct), text=PostingsFormat(name=LuceneVarGapDocFreqInterval), multiDefault=Lucene50(blocksize=128), timestamp=Lucene50(blocksize=128), id=PostingsFormat(name=Memory doPackFST= true), a_t=Lucene50(blocksize=128), _version_=PostingsFormat(name=Direct), range_facet_l=Lucene50(blocksize=128)}, docValues:{timestamp=DocValuesFormat(name=Lucene50)}, sim=RandomSimilarityProvider(queryNorm=false,coord=crazy): {}, locale=ar_LB, timezone=Canada/Newfoundland
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=66636552,total=305659904
   [junit4]   2> NOTE: All tests run in this JVM: [DistanceUnitsTest, TestSchemaVersionResource, DeleteReplicaTest, CoreAdminCreateDiscoverTest, TermsComponentTest, TestRandomFaceting, BadComponentTest, PolyFieldTest, IndexSchemaTest, TestCoreContainer, TestBulkSchemaConcurrent, TestWriterPerf, NoCacheHeaderTest, QueryEqualityTest, RecoveryZkTest, TestRecoveryHdfs, HdfsSyncSliceTest, DistribJoinFromCollectionTest, TestPivotHelperCode, DistributedSpellCheckComponentTest, TestBadConfig, SuggestComponentTest, TestManagedResourceStorage, RecoveryAfterSoftCommitTest, OpenExchangeRatesOrgProviderTest, SSLMigrationTest, DistanceFunctionTest, DistributedQueryElevationComponentTest, TestConfig, TestExactStatsCache, ScriptEngineTest, TestJmxMonitoredMap, TestConfigSets, SpellCheckCollatorTest, ResponseHeaderTest, HighlighterConfigTest, ResponseLogComponentTest, BasicFunctionalityTest, TestDFRSimilarityFactory, AddBlockUpdateTest, HardAutoCommitTest, ShardSplitTest, AsyncMigrateRouteKeyTest, TestSolrConfigHandlerConcurrent, AnalyticsQueryTest, EchoParamsTest, DirectUpdateHandlerOptimizeTest, SaslZkACLProviderTest, DistributedFacetPivotWhiteBoxTest, TestFieldResource, BJQParserTest, LeaderFailoverAfterPartitionTest, CursorPagingTest, MoreLikeThisHandlerTest, TestConfigOverlay, TestReloadAndDeleteDocs, TestClusterStateMutator, SpellCheckComponentTest, TestJmxIntegration, ReplicationFactorTest]
   [junit4] Completed on J1 in 76.76s, 1 test, 1 error <<< FAILURES!

[...truncated 721 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 1 failure, 61 ignored (21 assumptions)

Total time: 54 minutes 51 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 214173162 bytes
Compression is 0.0%
Took 43 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2887 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2887/

4 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:64721/xiwg/br/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:64721/xiwg/br/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([E317CCBEAFC08265:6B43F364013CEF9D]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:64721 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: 127.0.0.1:57112 failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:57112 failed to respond
	at __randomizedtesting.SeedInfo.seed([E317CCBEAFC08265:6B43F364013CEF9D]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:57112 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: 127.0.0.1:35138 failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:35138 failed to respond
	at __randomizedtesting.SeedInfo.seed([E317CCBEAFC08265:6B43F364013CEF9D]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:35138 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.handler.component.DistributedMLTComponentTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:59381//collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:59381//collection1
	at __randomizedtesting.SeedInfo.seed([E317CCBEAFC08265:6B43F364013CEF9D]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:566)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:943)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:558)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:606)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:588)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:567)
	at org.apache.solr.handler.component.DistributedMLTComponentTest.test(DistributedMLTComponentTest.java:126)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 51 more




Build Log:
[...truncated 9549 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/init-core-data-001
   [junit4]   2> 214318 T385 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /_/h
   [junit4]   2> 214326 T385 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 214327 T386 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 214427 T385 oasc.ZkTestServer.run start zk server on port:25045
   [junit4]   2> 214428 T385 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 214429 T385 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 214434 T393 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@719be136 name:ZooKeeperConnection Watcher:127.0.0.1:25045 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 214434 T385 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 214435 T385 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 214435 T385 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 214438 T385 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 214439 T385 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 214441 T396 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7b240a6a name:ZooKeeperConnection Watcher:127.0.0.1:25045/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 214441 T385 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 214441 T385 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 214442 T385 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 214444 T385 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 214446 T385 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 214447 T385 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 214449 T385 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 214449 T385 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 214452 T385 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 214452 T385 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 214454 T385 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 214455 T385 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 214456 T385 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 214457 T385 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 214458 T385 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 214459 T385 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 214460 T385 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 214460 T385 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 214462 T385 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 214462 T385 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 214464 T385 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 214464 T385 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 214466 T385 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 214466 T385 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 214468 T385 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 214468 T385 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 214469 T385 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 214470 T385 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 217283 T385 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/control-001/cores/collection1
   [junit4]   2> 217287 T385 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 217291 T385 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:27258
   [junit4]   2> 217291 T385 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/control-001/cores, hostContext=/_/h, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/tempDir-001/control/data, hostPort=31580}
   [junit4]   2> 217292 T385 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 217292 T385 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/control-001/'
   [junit4]   2> 217352 T385 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/control-001/solr.xml
   [junit4]   2> 217475 T385 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/control-001/cores
   [junit4]   2> 217475 T385 oasc.CoreContainer.<init> New CoreContainer 1201381070
   [junit4]   2> 217476 T385 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/control-001/]
   [junit4]   2> 217476 T385 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/control-001/lib
   [junit4]   2> 217478 T385 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/control-001/lib).
   [junit4]   2> 217493 T385 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 217493 T385 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 217494 T385 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 217494 T385 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 217494 T385 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 217495 T385 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 217495 T385 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 217495 T385 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 217496 T385 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 217496 T385 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 217496 T385 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 217509 T385 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 217510 T385 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 217511 T385 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 217511 T385 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 217512 T385 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:25045/solr
   [junit4]   2> 217512 T385 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 217513 T385 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 217514 T385 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 217516 T408 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@469f249f name:ZooKeeperConnection Watcher:127.0.0.1:25045 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 217517 T385 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 217518 T385 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 217521 T385 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 217523 T411 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@25ecfac5 name:ZooKeeperConnection Watcher:127.0.0.1:25045/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 217524 T385 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 217526 T385 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 217529 T385 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 217531 T385 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 217534 T385 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 217536 T385 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 217539 T385 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 217540 T385 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 217542 T385 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 217543 T385 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:31580__%2Fh
   [junit4]   2> 217544 T385 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:31580__%2Fh
   [junit4]   2> 217546 T385 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 217547 T385 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 217548 T385 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 217550 T385 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:31580__%2Fh
   [junit4]   2> 217550 T385 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 217552 T385 oasc.Overseer.start Overseer (id=93593343533056003-127.0.0.1:31580__%2Fh-n_0000000000) starting
   [junit4]   2> 217554 T385 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 217560 T385 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 217561 T413 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 217561 T385 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 217561 T412 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 217565 T385 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/control-001/cores
   [junit4]   2> 217569 T385 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {shard=, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/control-001/cores/collection1, collection=control_collection, schema=schema.xml, name=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/control-001/cores/collection1/, dataDir=data/, coreNodeName=, loadOnStartup=true, transient=false}
   [junit4]   2> 217570 T385 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/control-001/cores/collection1/
   [junit4]   2> 217570 T385 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 217571 T415 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 217572 T415 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 217573 T415 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 217573 T411 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 217574 T412 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:31580/_/h",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:31580__%2Fh"} current state version: 0
   [junit4]   2> 217575 T412 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:31580/_/h",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:31580__%2Fh"}
   [junit4]   2> 217575 T412 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 217576 T412 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 217577 T411 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 218573 T415 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 218574 T415 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 218574 T415 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 218575 T415 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 218575 T415 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 218576 T415 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/control-001/cores/collection1/'
   [junit4]   2> 218630 T415 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 218633 T415 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 218657 T415 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 218709 T415 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 218791 T415 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 218793 T415 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 218825 T415 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 219485 T415 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 219515 T415 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 219518 T415 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 219534 T415 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 219540 T415 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 219545 T415 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 219546 T415 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 219547 T415 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 219547 T415 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 219548 T415 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 219548 T415 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 219549 T415 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 219549 T415 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 219549 T415 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/control-001/cores/collection1/data/
   [junit4]   2> 219550 T415 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@62c5dc6a
   [junit4]   2> 219551 T415 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/control-001/cores/collection1/data
   [junit4]   2> 219551 T415 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/control-001/cores/collection1/data/index/
   [junit4]   2> 219551 T415 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 219552 T415 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/control-001/cores/collection1/data/index
   [junit4]   2> 219552 T415 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=24, maxMergeAtOnceExplicit=29, maxMergedSegmentMB=3.1484375, floorSegmentMB=1.59375, forceMergeDeletesPctAllowed=20.062224976109842, segmentsPerTier=43.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 219553 T415 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@57f15d5d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2728b1c6),segFN=segments_1,generation=1}
   [junit4]   2> 219554 T415 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 219562 T415 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 219562 T415 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 219563 T415 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 219563 T415 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 219563 T415 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 219564 T415 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 219564 T415 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 219565 T415 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 219565 T415 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 219608 T415 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 219611 T415 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 219614 T415 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 219616 T415 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 219624 T415 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /config,/admin/plugins,/get,/update/json/docs,/admin/ping,/update/csv,/schema,/admin/segments,/admin/system,/admin/file,/admin/mbeans,/admin/luke,standard,/admin/logging,/admin/threads,/update,/replication,/admin/properties,/update/json
   [junit4]   2> 219646 T415 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 219649 T415 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 219649 T415 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 219651 T415 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 219652 T415 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 219653 T415 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=37, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 219653 T415 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@57f15d5d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2728b1c6),segFN=segments_1,generation=1}
   [junit4]   2> 219654 T415 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 219654 T415 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@6648ff7e[collection1] main
   [junit4]   2> 219655 T415 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 219656 T415 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 219656 T415 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 219656 T415 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 219657 T415 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 219657 T415 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 219658 T415 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 219658 T415 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 219658 T415 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 219659 T415 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 219660 T416 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6648ff7e[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 219661 T415 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 219661 T419 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:31580/_/h collection:control_collection shard:shard1
   [junit4]   2> 219662 T385 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 219662 T385 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 219663 T419 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 219668 T419 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 219669 T385 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 219669 T411 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 219670 T419 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 219670 T419 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 219670 T385 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> ASYNC  NEW_CORE C63 name=collection1 org.apache.solr.core.SolrCore@63089797 url=http://127.0.0.1:31580/_/h/collection1 node=127.0.0.1:31580__%2Fh C63_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:31580/_/h, state=down, core=collection1, node_name=127.0.0.1:31580__%2Fh}
   [junit4]   2> 219670 T419 C:control_collection S:shard1 C63 P31580 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:31580/_/h/collection1/
   [junit4]   2> 219671 T419 C:control_collection S:shard1 C63 P31580 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 219671 T419 C:control_collection S:shard1 C63 P31580 oasc.SyncStrategy.syncToMe http://127.0.0.1:31580/_/h/collection1/ has no replicas
   [junit4]   2> 219672 T412 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 219672 T423 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@78d53fb8 name:ZooKeeperConnection Watcher:127.0.0.1:25045/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 219672 T419 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:31580/_/h/collection1/ shard1
   [junit4]   2> 219672 T385 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 219673 T419 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 219673 T411 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 219673 T385 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 219674 T385 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 219678 T411 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 219678 T385 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 219679 T412 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:31580/_/h",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 219782 T411 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 219782 T423 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 219830 T419 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:31580/_/h/collection1/ and leader is http://127.0.0.1:31580/_/h/collection1/
   [junit4]   2> 219831 T419 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:31580/_/h
   [junit4]   2> 219831 T419 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 219832 T419 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 219832 T419 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 219834 T411 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 219836 T412 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:31580/_/h",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:31580__%2Fh",
   [junit4]   2> 	  "core_node_name":"core_node1"} current state version: 3
   [junit4]   2> 219837 T412 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:31580/_/h",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:31580__%2Fh",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 220000 T423 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 220000 T411 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 222392 T385 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001/cores/collection1
   [junit4]   2> 222397 T385 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001
   [junit4]   2> 222399 T385 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 222403 T385 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:59242
   [junit4]   2> 222403 T385 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/_/h, solrconfig=solrconfig.xml, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/tempDir-001/jetty1, hostPort=49825, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001/cores}
   [junit4]   2> 222404 T385 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 222404 T385 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001/'
   [junit4]   2> 222483 T385 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001/solr.xml
   [junit4]   2> 222583 T385 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001/cores
   [junit4]   2> 222584 T385 oasc.CoreContainer.<init> New CoreContainer 1298212678
   [junit4]   2> 222584 T385 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001/]
   [junit4]   2> 222585 T385 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001/lib
   [junit4]   2> 222586 T385 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001/lib).
   [junit4]   2> 222602 T385 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 222603 T385 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 222603 T385 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 222604 T385 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 222604 T385 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 222604 T385 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 222605 T385 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 222605 T385 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 222606 T385 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 222606 T385 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 222606 T385 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 222630 T385 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 222632 T385 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 222632 T385 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 222633 T385 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 222633 T385 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:25045/solr
   [junit4]   2> 222633 T385 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 222634 T385 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 222635 T385 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 222637 T435 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1962080 name:ZooKeeperConnection Watcher:127.0.0.1:25045 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 222638 T385 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 222639 T385 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 222642 T385 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 222643 T438 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@373bc8de name:ZooKeeperConnection Watcher:127.0.0.1:25045/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 222644 T385 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 222649 T385 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 223654 T385 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49825__%2Fh
   [junit4]   2> 223655 T385 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49825__%2Fh
   [junit4]   2> 223658 T385 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 223661 T385 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001/cores
   [junit4]   2> 223664 T385 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {transient=false, name=collection1, coreNodeName=, config=solrconfig.xml, collection=collection1, schema=schema.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001/cores/collection1/, loadOnStartup=true, shard=, dataDir=data/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001/cores/collection1}
   [junit4]   2> 223665 T385 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001/cores/collection1/
   [junit4]   2> 223665 T385 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 223667 T439 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 223667 T439 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 223668 T439 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 223668 T411 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 223670 T412 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49825/_/h",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:49825__%2Fh"} current state version: 4
   [junit4]   2> 223670 T412 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49825/_/h",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:49825__%2Fh"}
   [junit4]   2> 223671 T412 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 223671 T412 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 223673 T438 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 223673 T423 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 223673 T411 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 224669 T439 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 224670 T439 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 224671 T439 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 224671 T439 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 224672 T439 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 224672 T439 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001/cores/collection1/'
   [junit4]   2> 224697 T439 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 224700 T439 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 224717 T439 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 224761 T439 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 224842 T439 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 224843 T439 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 224859 T439 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 225486 T439 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 225511 T439 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 225515 T439 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 225530 T439 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 225537 T439 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 225542 T439 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 225543 T439 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 225544 T439 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 225544 T439 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 225545 T439 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 225545 T439 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 225546 T439 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 225546 T439 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 225546 T439 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 225547 T439 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@62c5dc6a
   [junit4]   2> 225548 T439 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001/cores/collection1/data
   [junit4]   2> 225548 T439 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 225549 T439 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 225549 T439 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest E317CCBEAFC08265-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 225550 T439 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=24, maxMergeAtOnceExplicit=29, maxMergedSegmentMB=3.1484375, floorSegmentMB=1.59375, forceMergeDeletesPctAllowed=20.062224976109842, segmentsPerTier=43.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 225551 T439 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6ab86bf4 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@71b796e8),segFN=segments_1,generation=1}
   [junit4]   2> 225551 T439 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 225562 T439 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 225562 T439 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 225563 T439 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 225563 T439 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 225563 T439 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 225564 T439 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 225564 T439 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 225565 T439 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 225565 T439 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 225608 T439 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 225612 T439 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 225617 T439 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 225622 T439 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 225635 T439 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /config,/admin/plugins,/get,/update/json/docs,/admin/ping,/update/csv,/schema,/admin/segments,/admin/system,/admin/file,/admin/mbeans,/admin/luke,standard,/admin/logging,/admin/threads,/update,/replication,/admin/properties,/update/json
   [junit4]   2> 225638 T439 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 225641 T439 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 225641 T439 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 225644 T439 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 225644 T439 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 225645 T439 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=37, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 225646 T439 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6ab86bf4 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@71b796e8),segFN=segments_1,generation=1}
   [junit4]   2> 225647 T439 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 225647 T439 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@4be308fe[collection1] main
   [junit4]   2> 225648 T439 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 225649 T439 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 225649 T439 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 225650 T439 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 225650 T439 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 225650 T439 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 225651 T439 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 225651 T439 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 225652 T439 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 225652 T439 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 225654 T440 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4be308fe[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 225655 T439 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 225656 T443 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:49825/_/h collection:collection1 shard:shard1
   [junit4]   2> 225657 T385 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 225657 T385 oass.SolrDispatchFilter.init So

[...truncated too long message...]

it4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1940494 T10737 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:57116/
   [junit4]   2> 1940494 T10737 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:57112/
   [junit4]   2> 1940494 T10737 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:57107/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=E317CCBEAFC08265 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=it_IT -Dtests.timezone=America/Toronto -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   67.9s J3 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:57112 failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([E317CCBEAFC08265:6B43F364013CEF9D]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:57112 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 1940511 T10737 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest E317CCBEAFC08265-001
   [junit4]   2> 67949 T10736 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {id=PostingsFormat(name=Asserting), a_t=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), _version_=PostingsFormat(name=Memory doPackFST= true), range_facet_l=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), text=PostingsFormat(name=LuceneVarGapDocFreqInterval), timestamp=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), multiDefault=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), intDefault=PostingsFormat(name=Memory doPackFST= true)}, docValues:{timestamp=DocValuesFormat(name=Memory)}, sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {}, locale=it_IT, timezone=America/Toronto
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=103991192,total=342884352
   [junit4]   2> NOTE: All tests run in this JVM: [AddSchemaFieldsUpdateProcessorFactoryTest, TestSimpleTrackingShardHandler, TermsComponentTest, TestPerFieldSimilarity, TestBM25SimilarityFactory, TestImplicitCoreProperties, DocValuesMissingTest, TestDefaultSearchFieldResource, CSVRequestHandlerTest, SortByFunctionTest, CoreAdminRequestStatusTest, MergeStrategyTest, TestFastOutputStream, OpenCloseCoreStressTest, TestPivotHelperCode, RequestLoggingTest, UniqFieldsUpdateProcessorFactoryTest, RequestHandlersTest, StatelessScriptUpdateProcessorFactoryTest, TestStandardQParsers, TestShardHandlerFactory, SchemaVersionSpecificBehaviorTest, TestConfigOverlay, TestFieldSortValues, TestReloadAndDeleteDocs, TestSolrConfigHandler, AtomicUpdatesTest, DistribCursorPagingTest, SignatureUpdateProcessorFactoryTest, TestReRankQParserPlugin, TestBinaryField, TestReversedWildcardFilterFactory, CollectionsAPIDistributedZkTest, DistributedQueryElevationComponentTest, DocValuesTest, SuggesterTest, LeaderInitiatedRecoveryOnCommitTest, TestConfigSets, FullSolrCloudDistribCmdsTest, TestUpdate, ZkSolrClientTest, TestManagedSchema, FieldAnalysisRequestHandlerTest, TestCustomSort, ExternalCollectionsTest, SharedFSAutoReplicaFailoverUtilsTest, TestManagedStopFilterFactory, TestMacros, OutputWriterTest, DistributedMLTComponentTest, TestDynamicLoading, CoreMergeIndexesAdminHandlerTest, TestCSVResponseWriter, TestStressReorder, TestManagedSchemaDynamicFieldResource, ClusterStateUpdateTest, DirectUpdateHandlerOptimizeTest, URLClassifyProcessorTest, DistributedFacetPivotSmallTest, TestFiltering, BasicDistributedZk2Test, BasicDistributedZkTest, TestSolrConfigHandlerCloud, LeaderElectionIntegrationTest, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J3 in 68.48s, 1 test, 1 error <<< FAILURES!

[...truncated 709 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 4 errors, 61 ignored (21 assumptions)

Total time: 60 minutes 36 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 208692924 bytes
Compression is 0.0%
Took 43 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2886 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2886/

1 tests failed.
REGRESSION:  org.apache.lucene.analysis.uima.UIMABaseAnalyzerTest.testRandomStringsWithConfigurationParameters

Error Message:
some thread(s) failed

Stack Trace:
java.lang.RuntimeException: some thread(s) failed
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:531)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:428)
	at org.apache.lucene.analysis.uima.UIMABaseAnalyzerTest.testRandomStringsWithConfigurationParameters(UIMABaseAnalyzerTest.java:133)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 3939 lines...]
   [junit4] Suite: org.apache.lucene.analysis.uima.UIMABaseAnalyzerTest
   [junit4]   2> Abr 03, 2015 8:02:12 PM WhitespaceTokenizer initialize
   [junit4]   2> INFO: "Whitespace tokenizer successfully initialized"
   [junit4]   2> Abr 03, 2015 8:02:12 PM WhitespaceTokenizer typeSystemInit
   [junit4]   2> INFO: "Whitespace tokenizer typesystem initialized"
   [junit4]   2> Abr 03, 2015 8:02:12 PM WhitespaceTokenizer process
   [junit4]   2> INFO: "Whitespace tokenizer starts processing"
   [junit4]   2> Abr 03, 2015 8:02:12 PM WhitespaceTokenizer process
   [junit4]   2> INFO: "Whitespace tokenizer finished processing"
   [junit4]   2> Abr 03, 2015 8:02:12 PM WhitespaceTokenizer process
   [junit4]   2> INFO: "Whitespace tokenizer starts processing"
   [junit4]   2> Abr 03, 2015 8:02:12 PM WhitespaceTokenizer process
   [junit4]   2> INFO: "Whitespace tokenizer finished processing"
   [junit4]   2> Abr 03, 2015 8:02:13 PM WhitespaceTokenizer process
   [junit4]   2> INFO: "Whitespace tokenizer starts processing"
   [junit4]   2> Abr 03, 2015 8:02:13 PM WhitespaceTokenizer process
   [junit4]   2> INFO: "Whitespace tokenizer finished processing"
   [junit4]   2> Abr 03, 2015 8:02:13 PM WhitespaceTokenizer process
   [junit4]   2> INFO: "Whitespace tokenizer starts processing"
   [junit4]   2> Abr 03, 2015 8:02:13 PM WhitespaceTokenizer process
   [junit4]   2> INFO: "Whitespace tokenizer finished processing"
   [junit4]   2> TEST FAIL: useCharFilter=false text='yej wbw  xeln \u0e97\u0ed4\u0e80'
   [junit4]   2> Abr 03, 2015 8:02:14 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: Thread[Thread-8,5,TGRP-UIMABaseAnalyzerTest]
   [junit4]   2> java.lang.AssertionError
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([5C1B564E414DC109]:0)
   [junit4]   2> 	at sun.reflect.generics.reflectiveObjects.WildcardTypeImpl.getLowerBoundASTs(WildcardTypeImpl.java:94)
   [junit4]   2> 	at sun.reflect.generics.reflectiveObjects.WildcardTypeImpl.getLowerBounds(WildcardTypeImpl.java:165)
   [junit4]   2> 	at com.sun.beans.TypeResolver.resolve(TypeResolver.java:176)
   [junit4]   2> 	at com.sun.beans.TypeResolver.resolve(TypeResolver.java:218)
   [junit4]   2> 	at com.sun.beans.TypeResolver.resolve(TypeResolver.java:169)
   [junit4]   2> 	at com.sun.beans.TypeResolver.resolveInClass(TypeResolver.java:81)
   [junit4]   2> 	at java.beans.FeatureDescriptor.getReturnType(FeatureDescriptor.java:370)
   [junit4]   2> 	at java.beans.Introspector.getTargetEventInfo(Introspector.java:996)
   [junit4]   2> 	at java.beans.Introspector.getBeanInfo(Introspector.java:417)
   [junit4]   2> 	at java.beans.Introspector.getBeanInfo(Introspector.java:252)
   [junit4]   2> 	at java.beans.Introspector.<init>(Introspector.java:397)
   [junit4]   2> 	at java.beans.Introspector.getBeanInfo(Introspector.java:252)
   [junit4]   2> 	at java.beans.Introspector.<init>(Introspector.java:397)
   [junit4]   2> 	at java.beans.Introspector.getBeanInfo(Introspector.java:252)
   [junit4]   2> 	at java.beans.Introspector.getBeanInfo(Introspector.java:194)
   [junit4]   2> 	at org.apache.uima.resource.metadata.impl.MetaDataObject_impl.getPropertyDescriptors(MetaDataObject_impl.java:1269)
   [junit4]   2> 	at org.apache.uima.resource.metadata.impl.MetaDataObject_impl.listAttributes(MetaDataObject_impl.java:105)
   [junit4]   2> 	at org.apache.uima.resource.metadata.impl.MetaDataObject_impl.setSourceUrl(MetaDataObject_impl.java:289)
   [junit4]   2> 	at org.apache.uima.util.CasCreationUtils.addTypeToMergedTypeSystem(CasCreationUtils.java:1125)
   [junit4]   2> 	at org.apache.uima.util.CasCreationUtils.mergeTypeSystems(CasCreationUtils.java:1090)
   [junit4]   2> 	at org.apache.uima.util.CasCreationUtils.mergeTypeSystems(CasCreationUtils.java:1019)
   [junit4]   2> 	at org.apache.uima.resource.CasDefinition.<init>(CasDefinition.java:78)
   [junit4]   2> 	at org.apache.uima.resource.impl.CasManager_impl.getCasDefinition(CasManager_impl.java:94)
   [junit4]   2> 	at org.apache.uima.analysis_engine.impl.AnalysisEngineImplBase.newCAS(AnalysisEngineImplBase.java:230)
   [junit4]   2> 	at org.apache.lucene.analysis.uima.BaseUIMATokenizer.analyzeInput(BaseUIMATokenizer.java:67)
   [junit4]   2> 	at org.apache.lucene.analysis.uima.UIMAAnnotationsTokenizer.initializeIterator(UIMAAnnotationsTokenizer.java:60)
   [junit4]   2> 	at org.apache.lucene.analysis.uima.UIMAAnnotationsTokenizer.incrementToken(UIMAAnnotationsTokenizer.java:72)
   [junit4]   2> 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkAnalysisConsistency(BaseTokenStreamTestCase.java:700)
   [junit4]   2> 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:611)
   [junit4]   2> 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.access$000(BaseTokenStreamTestCase.java:60)
   [junit4]   2> 	at org.apache.lucene.analysis.BaseTokenStreamTestCase$AnalysisThread.run(BaseTokenStreamTestCase.java:479)
   [junit4]   2> 
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=UIMABaseAnalyzerTest -Dtests.method=testRandomStringsWithConfigurationParameters -Dtests.seed=5C1B564E414DC109 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=pt_PT -Dtests.timezone=SystemV/AST4ADT -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   1.52s | UIMABaseAnalyzerTest.testRandomStringsWithConfigurationParameters <<<
   [junit4]    > Throwable #1: java.lang.RuntimeException: some thread(s) failed
   [junit4]    > 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:531)
   [junit4]    > 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:428)
   [junit4]    > 	at org.apache.lucene.analysis.uima.UIMABaseAnalyzerTest.testRandomStringsWithConfigurationParameters(UIMABaseAnalyzerTest.java:133)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #2: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=19, name=Thread-8, state=RUNNABLE, group=TGRP-UIMABaseAnalyzerTest]
   [junit4]    > Caused by: java.lang.AssertionError
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([5C1B564E414DC109]:0)
   [junit4]    > 	at com.sun.beans.TypeResolver.resolve(TypeResolver.java:176)
   [junit4]    > 	at com.sun.beans.TypeResolver.resolve(TypeResolver.java:218)
   [junit4]    > 	at com.sun.beans.TypeResolver.resolve(TypeResolver.java:169)
   [junit4]    > 	at com.sun.beans.TypeResolver.resolveInClass(TypeResolver.java:81)
   [junit4]    > 	at java.beans.FeatureDescriptor.getReturnType(FeatureDescriptor.java:370)
   [junit4]    > 	at java.beans.Introspector.getTargetEventInfo(Introspector.java:996)
   [junit4]    > 	at java.beans.Introspector.getBeanInfo(Introspector.java:417)
   [junit4]    > 	at java.beans.Introspector.getBeanInfo(Introspector.java:252)
   [junit4]    > 	at java.beans.Introspector.<init>(Introspector.java:397)
   [junit4]    > 	at java.beans.Introspector.getBeanInfo(Introspector.java:252)
   [junit4]    > 	at java.beans.Introspector.<init>(Introspector.java:397)
   [junit4]    > 	at java.beans.Introspector.getBeanInfo(Introspector.java:252)
   [junit4]    > 	at java.beans.Introspector.getBeanInfo(Introspector.java:194)
   [junit4]    > 	at org.apache.uima.resource.metadata.impl.MetaDataObject_impl.getPropertyDescriptors(MetaDataObject_impl.java:1269)
   [junit4]    > 	at org.apache.uima.resource.metadata.impl.MetaDataObject_impl.listAttributes(MetaDataObject_impl.java:105)
   [junit4]    > 	at org.apache.uima.resource.metadata.impl.MetaDataObject_impl.setSourceUrl(MetaDataObject_impl.java:289)
   [junit4]    > 	at org.apache.uima.util.CasCreationUtils.addTypeToMergedTypeSystem(CasCreationUtils.java:1125)
   [junit4]    > 	at org.apache.uima.util.CasCreationUtils.mergeTypeSystems(CasCreationUtils.java:1090)
   [junit4]    > 	at org.apache.uima.util.CasCreationUtils.mergeTypeSystems(CasCreationUtils.java:1019)
   [junit4]    > 	at org.apache.uima.resource.CasDefinition.<init>(CasDefinition.java:78)
   [junit4]    > 	at org.apache.uima.resource.impl.CasManager_impl.getCasDefinition(CasManager_impl.java:94)
   [junit4]    > 	at org.apache.uima.analysis_engine.impl.AnalysisEngineImplBase.newCAS(AnalysisEngineImplBase.java:230)
   [junit4]    > 	at org.apache.lucene.analysis.uima.BaseUIMATokenizer.analyzeInput(BaseUIMATokenizer.java:67)
   [junit4]    > 	at org.apache.lucene.analysis.uima.UIMAAnnotationsTokenizer.initializeIterator(UIMAAnnotationsTokenizer.java:60)
   [junit4]    > 	at org.apache.lucene.analysis.uima.UIMAAnnotationsTokenizer.incrementToken(UIMAAnnotationsTokenizer.java:72)
   [junit4]    > 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkAnalysisConsistency(BaseTokenStreamTestCase.java:700)
   [junit4]    > 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:611)
   [junit4]    > 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.access$000(BaseTokenStreamTestCase.java:60)
   [junit4]    > 	at org.apache.lucene.analysis.BaseTokenStreamTestCase$AnalysisThread.run(BaseTokenStreamTestCase.java:479)
   [junit4]   2> Abr 03, 2015 8:02:18 PM WhitespaceTokenizer initialize
   [junit4]   2> INFO: "Whitespace tokenizer successfully initialized"
   [junit4]   2> Abr 03, 2015 8:02:18 PM WhitespaceTokenizer typeSystemInit
   [junit4]   2> INFO: "Whitespace tokenizer typesystem initialized"
   [junit4]   2> Abr 03, 2015 8:02:18 PM WhitespaceTokenizer process
   [junit4]   2> INFO: "Whitespace tokenizer starts processing"
   [junit4]   2> Abr 03, 2015 8:02:18 PM WhitespaceTokenizer process
   [junit4]   2> INFO: "Whitespace tokenizer finished processing"
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {dummy=FSTOrd50, title=FSTOrd50, contents=FSTOrd50}, docValues:{}, sim=DefaultSimilarity, locale=pt_PT, timezone=SystemV/AST4ADT
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=122198424,total=301465600
   [junit4]   2> NOTE: All tests run in this JVM: [UIMABaseAnalyzerTest]
   [junit4] Completed in 11.63s, 4 tests, 1 error <<< FAILURES!

[...truncated 18 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/build.xml:456: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:2166: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/analysis/build.xml:106: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/analysis/build.xml:38: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/module-build.xml:58: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 5 suites, 12 tests, 1 error

Total time: 8 minutes 20 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 2 artifacts
Archive block size is 32768
Received 0 blocks and 54261 bytes
Compression is 0.0%
Took 0.4 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2885 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2885/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:16808/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:16808/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([8632F28962AA5D4B:E66CD53CC5630B3]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:16808 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:65051/jn_qr/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:65051/jn_qr/collection1
	at __randomizedtesting.SeedInfo.seed([8632F28962AA5D4B:E66CD53CC5630B3]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:65051/jn_qr/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:65051 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:10434/i/p/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:10434/i/p/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([8632F28962AA5D4B:E66CD53CC5630B3]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:10434/i/p/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:10434 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9407 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/init-core-data-001
   [junit4]   2> 190587 T677 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /jn_qr/
   [junit4]   2> 190592 T677 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 190593 T678 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 190693 T677 oasc.ZkTestServer.run start zk server on port:65022
   [junit4]   2> 190694 T677 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 190695 T677 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 190699 T685 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@302109de name:ZooKeeperConnection Watcher:127.0.0.1:65022 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 190699 T677 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 190700 T677 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 190700 T677 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 190703 T677 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 190704 T677 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 190706 T688 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@75a792d8 name:ZooKeeperConnection Watcher:127.0.0.1:65022/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 190706 T677 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 190707 T677 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 190707 T677 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 190709 T677 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 190711 T677 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 190713 T677 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 190715 T677 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 190716 T677 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 190719 T677 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 190720 T677 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 190823 T677 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 190823 T677 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 190825 T677 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 190826 T677 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 190827 T677 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 190828 T677 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 190830 T677 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 190830 T677 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 190832 T677 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 190832 T677 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 190834 T677 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 190834 T677 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 190836 T677 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 190836 T677 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 190838 T677 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 190839 T677 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 190841 T677 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 190841 T677 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 192005 T677 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/control-001/cores/collection1
   [junit4]   2> 192010 T677 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 192014 T677 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:65038
   [junit4]   2> 192015 T677 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/jn_qr, hostPort=65037, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/tempDir-001/control/data}
   [junit4]   2> 192016 T677 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 192016 T677 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/control-001/'
   [junit4]   2> 192065 T677 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/control-001/solr.xml
   [junit4]   2> 192128 T677 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/control-001/cores
   [junit4]   2> 192129 T677 oasc.CoreContainer.<init> New CoreContainer 359853771
   [junit4]   2> 192129 T677 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/control-001/]
   [junit4]   2> 192130 T677 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/control-001/lib
   [junit4]   2> 192131 T677 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/control-001/lib).
   [junit4]   2> 192145 T677 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 192146 T677 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 192146 T677 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 192146 T677 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 192147 T677 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 192147 T677 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 192148 T677 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 192148 T677 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 192148 T677 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 192149 T677 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 192149 T677 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 192157 T677 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 192159 T677 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 192159 T677 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 192159 T677 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 192160 T677 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:65022/solr
   [junit4]   2> 192161 T677 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 192161 T677 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 192162 T677 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 192164 T700 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4da90726 name:ZooKeeperConnection Watcher:127.0.0.1:65022 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 192164 T677 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 192165 T677 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 192167 T677 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 192169 T703 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@470d25cb name:ZooKeeperConnection Watcher:127.0.0.1:65022/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 192169 T677 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 192171 T677 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 192174 T677 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 192176 T677 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 192179 T677 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 192182 T677 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 192184 T677 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 192186 T677 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 192188 T677 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 192190 T677 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:65037_jn_qr
   [junit4]   2> 192191 T677 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:65037_jn_qr
   [junit4]   2> 192193 T677 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 192194 T677 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 192196 T677 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 192198 T677 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:65037_jn_qr
   [junit4]   2> 192198 T677 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 192200 T677 oasc.Overseer.start Overseer (id=93591450468089859-127.0.0.1:65037_jn_qr-n_0000000000) starting
   [junit4]   2> 192202 T677 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 192210 T677 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 192211 T705 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 192211 T677 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 192211 T704 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 192216 T677 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/control-001/cores
   [junit4]   2> 192218 T677 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, config=solrconfig.xml, collection=control_collection, schema=schema.xml, dataDir=data/, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/control-001/cores/collection1/, transient=false, loadOnStartup=true, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/control-001/cores/collection1, shard=, coreNodeName=}
   [junit4]   2> 192219 T677 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/control-001/cores/collection1/
   [junit4]   2> 192219 T677 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 192221 T707 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 192221 T707 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 192222 T703 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 192222 T707 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 192223 T704 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:65037/jn_qr",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:65037_jn_qr",
   [junit4]   2> 	  "roles":null} current state version: 0
   [junit4]   2> 192224 T704 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:65037/jn_qr",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:65037_jn_qr",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 192224 T704 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 192225 T704 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 192226 T703 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 193223 T707 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 193224 T707 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 193225 T707 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 193225 T707 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 193225 T707 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 193226 T707 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/control-001/cores/collection1/'
   [junit4]   2> 193240 T707 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 193242 T707 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 193251 T707 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 193271 T707 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 193310 T707 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 193411 T707 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 193421 T707 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 193988 T707 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 194002 T707 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 194004 T707 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 194065 T707 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 194070 T707 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 194075 T707 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 194076 T707 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 194076 T707 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 194076 T707 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 194077 T707 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 194077 T707 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 194078 T707 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 194078 T707 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 194078 T707 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/control-001/cores/collection1/data/
   [junit4]   2> 194079 T707 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@118c55d6
   [junit4]   2> 194079 T707 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/control-001/cores/collection1/data
   [junit4]   2> 194080 T707 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/control-001/cores/collection1/data/index/
   [junit4]   2> 194080 T707 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 194080 T707 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/control-001/cores/collection1/data/index
   [junit4]   2> 194081 T707 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=11, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 194081 T707 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@41dc3a24 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@616d3343),segFN=segments_1,generation=1}
   [junit4]   2> 194082 T707 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 194087 T707 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 194088 T707 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 194088 T707 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 194088 T707 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 194088 T707 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 194089 T707 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 194089 T707 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 194089 T707 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 194089 T707 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 194110 T707 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 194113 T707 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 194115 T707 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 194117 T707 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 194123 T707 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/plugins,/admin/system,/replication,/admin/ping,/admin/segments,/update/csv,/schema,/admin/properties,/admin/threads,/config,/update/json/docs,/admin/file,standard,/admin/logging,/update/json,/update,/admin/luke,/admin/mbeans,/get
   [junit4]   2> 194127 T707 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 194128 T707 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 194129 T707 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 194130 T707 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 194130 T707 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 194131 T707 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=29, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=59.6513671875, floorSegmentMB=1.1591796875, forceMergeDeletesPctAllowed=20.822084393283095, segmentsPerTier=50.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 194131 T707 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@41dc3a24 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@616d3343),segFN=segments_1,generation=1}
   [junit4]   2> 194132 T707 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 194132 T707 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@4d36517d[collection1] main
   [junit4]   2> 194132 T707 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 194133 T707 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 194133 T707 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 194134 T707 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 194134 T707 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 194134 T707 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 194135 T707 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 194135 T707 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 194135 T707 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 194136 T707 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 194136 T708 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4d36517d[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 194137 T707 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 194138 T711 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:65037/jn_qr collection:control_collection shard:shard1
   [junit4]   2> 194138 T677 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 194138 T677 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 194139 T711 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 194142 T677 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 194143 T677 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 194143 T711 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 194144 T715 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2f3e08d5 name:ZooKeeperConnection Watcher:127.0.0.1:65022/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 194145 T677 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 194145 T677 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 194145 T703 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 194145 T711 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 194146 T711 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 194145 T677 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 194146 T704 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C50 name=collection1 org.apache.solr.core.SolrCore@79ed71bd url=http://127.0.0.1:65037/jn_qr/collection1 node=127.0.0.1:65037_jn_qr C50_STATE=coll:control_collection core:collection1 props:{state=down, core=collection1, base_url=http://127.0.0.1:65037/jn_qr, node_name=127.0.0.1:65037_jn_qr}
   [junit4]   2> 194146 T711 C:control_collection S:shard1 C50 P65037 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:65037/jn_qr/collection1/
   [junit4]   2> 194147 T711 C:control_collection S:shard1 C50 P65037 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 194147 T711 C:control_collection S:shard1 C50 P65037 oasc.SyncStrategy.syncToMe http://127.0.0.1:65037/jn_qr/collection1/ has no replicas
   [junit4]   2> 194147 T711 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:65037/jn_qr/collection1/ shard1
   [junit4]   2> 194147 T711 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 194148 T715 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 194148 T703 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 194150 T677 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 194152 T703 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 194153 T704 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:65037/jn_qr",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 194256 T715 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 194256 T703 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 194305 T711 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:65037/jn_qr/collection1/ and leader is http://127.0.0.1:65037/jn_qr/collection1/
   [junit4]   2> 194306 T711 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:65037/jn_qr
   [junit4]   2> 194306 T711 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 194306 T711 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 194307 T711 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 194308 T703 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 194310 T704 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:65037/jn_qr",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:65037_jn_qr",
   [junit4]   2> 	  "roles":null} current state version: 3
   [junit4]   2> 194310 T704 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:65037/jn_qr",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:65037_jn_qr",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 194414 T715 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 194414 T703 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 195467 T677 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001/cores/collection1
   [junit4]   2> 195469 T677 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001
   [junit4]   2> 195470 T677 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 195472 T677 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:65052
   [junit4]   2> 195473 T677 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/jn_qr, hostPort=65051, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001/cores}
   [junit4]   2> 195473 T677 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 195473 T677 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001/'
   [junit4]   2> 195509 T677 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001/solr.xml
   [junit4]   2> 195547 T677 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001/cores
   [junit4]   2> 195547 T677 oasc.CoreContainer.<init> New CoreContainer 670326152
   [junit4]   2> 195548 T677 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001/]
   [junit4]   2> 195548 T677 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001/lib
   [junit4]   2> 195548 T677 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001/lib).
   [junit4]   2> 195558 T677 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 195559 T677 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 195559 T677 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 195559 T677 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 195559 T677 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 195560 T677 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 195560 T677 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 195560 T677 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 195561 T677 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 195561 T677 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 195561 T677 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 195566 T677 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 195567 T677 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 195568 T677 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 195568 T677 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 195568 T677 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:65022/solr
   [junit4]   2> 195568 T677 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 195569 T677 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 195569 T677 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 195571 T727 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5ca40bc7 name:ZooKeeperConnection Watcher:127.0.0.1:65022 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 195572 T677 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 195573 T677 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 195574 T677 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 195576 T730 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@360d8c9a name:ZooKeeperConnection Watcher:127.0.0.1:65022/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 195576 T677 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 195582 T677 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 196587 T677 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:65051_jn_qr
   [junit4]   2> 196588 T677 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:65051_jn_qr
   [junit4]   2> 196592 T677 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 196595 T677 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001/cores
   [junit4]   2> 196598 T677 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {schema=schema.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001/cores/collection1/, transient=false, config=solrconfig.xml, shard=, dataDir=data/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001/cores/collection1, collection=collection1, name=collection1, coreNodeName=, loadOnStartup=true}
   [junit4]   2> 196598 T677 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001/cores/collection1/
   [junit4]   2> 196599 T677 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 196600 T731 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 196601 T731 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 196602 T703 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 196602 T731 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 196603 T704 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:65051/jn_qr",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:65051_jn_qr",
   [junit4]   2> 	  "roles":null} current state version: 4
   [junit4]   2> 196604 T704 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:65051/jn_qr",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:65051_jn_qr",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 196604 T704 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 196604 T704 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 196606 T730 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 196606 T703 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 196606 T715 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 197603 T731 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 197604 T731 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 197604 T731 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 197604 T731 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 197605 T731 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 197606 T731 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001/cores/collection1/'
   [junit4]   2> 197621 T731 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 197624 T731 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 197635 T731 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 197657 T731 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 197703 T731 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 197805 T731 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 197816 T731 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 198272 T731 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 198293 T731 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 198297 T731 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 198318 T731 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 198324 T731 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 198330 T731 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 198331 T731 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 198331 T731 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 198332 T731 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 198333 T731 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 198333 T731 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 198334 T731 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 198334 T731 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 198334 T731 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 198335 T731 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@118c55d6
   [junit4]   2> 198336 T731 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001/cores/collection1/data
   [junit4]   2> 198336 T731 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 198337 T731 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 198337 T731 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8632F28962AA5D4B-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 198338 T731 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=11, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 198339 T731 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3e14f86f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@52c25f40),segFN=segments_1,generation=1}
   [junit4]   2> 198339 T731 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 198350 T731 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 198351 T731 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 198351 T731 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 198352 T731 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 198352 T731 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 198352 T731 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 198354 T731 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 198354 T731 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 198354 T731 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 198389 T731 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 198392 T731 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 198395 T731 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 198398 T731 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 198408 T731 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/plugins,/admin/system,/replication,/admin/ping,/admin/segments,/update/csv,/schema,/admin/properties,/admin/threads,/config,/update/json/docs,/admin/file,standard,/admin/logging,/update/json,/update,/admin/luke,/admin/mbeans,/get
   [junit4]   2> 198411 T731 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 198413 T731 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 198414 T731 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 198416 T731 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 198416 T731 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 198417 T731 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=29, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=59.6513671875, floorSegmentMB=1.1591796875, forceMergeDeletesPctAllowed=20.822084393283095, segmentsPerTier=50.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 198418 T731 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3e14f86f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@52c25f40),segFN=segments_1,generation=1}
   [junit4]   2> 198418 T731 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 198419 T731 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@143272e2[collection1] main
   [junit4]   2> 198419 T731 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 198420 T731 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 198421 T731 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 198421 T731 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 198422 T731 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 198422 T731 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 198423 T731 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 198423 T731 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 198423 T731 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 198424 T731 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 198425 T732 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@143272e2[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 198426 T731 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 198427 T735 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:65051/jn_qr collection:collection1 shard:shard1
   [junit4]   2> 198427 T677 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 198428 T677 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 198428 T735 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 198435 T735 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 198437 T703 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 198437 T735 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 198438 T735 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C51 name=collection1 org.apache.solr.core.SolrCore@5fa8113e url=http://127.0.0.1:65051/jn_qr/collection1 node=127.0.0.1:65051_jn_qr C51_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, base_url=http://127.0.0.1:65051/jn_qr, node_name=127.0.0.1:65051_jn_qr}
   [junit4]   2> 198438 T735 C:collection1 S:shard1 C51 P65051 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:65051/jn_qr/collection1/
   [junit4]   2> 198439 T704 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 198439 T735 C:collection1 S:shard1 C51 P65051 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 198439 T735 C:collection1 S:shard1 C51 P65051 oasc.SyncStrategy.syncToMe http://127.0.0.1:65051/jn_qr/collection1/ has no replicas
   [junit4]   2> 198440 T735 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:65051/jn_qr/collection1/ shard1
   [junit4]   2> 198440 T735 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 198441 T703 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 198441 T730 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 198441 T715 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 198446 T703 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 198447 T704 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:65051/jn_qr",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 198552 T703 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 198552 T730 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 198552 T715 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 198599 T735 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:65051/jn_qr/collection1/ and leader is http://127.0.0.1:65051/jn_qr/collection1/
   [junit4]   2> 198600 T735 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:65051/jn_qr
   [junit4]   2> 198600 T735 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 198601 T735 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 198601 T735 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 198603 T703 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 198604 T704 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:65051/jn_qr",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:65051_jn_qr",
   [junit4]   2> 	  "roles":null} current state version: 7
   [junit4]   2> 198605 T704 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:65051/jn_qr",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:65051_jn_qr",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 198710 T703 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 198710 T715 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 198710 T730 oascc.ZkSta

[...truncated too long message...]

unit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/aliases.json
   [junit4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 		3	/solr/collections/c8n_1x3_commits/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent data watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		2	/solr/collections/c8n_1x3_commits/leader_elect/shard1/election/93591532682346506-core_node2-n_0000000000
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 		4	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1524390 T9440 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16818/
   [junit4]   2> 1524390 T9440 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16812/
   [junit4]   2> 1524391 T9440 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 13 connections to: http://127.0.0.1:16808/
   [junit4]   2> 1524391 T9440 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16832/
   [junit4]   2> 1524391 T9440 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 3 connections to: http://127.0.0.1:16800/
   [junit4]   2> 1524391 T9440 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16818/
   [junit4]   2> 1524392 T9440 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16812/
   [junit4]   2> 1524392 T9440 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16808/
   [junit4]   2> 1524392 T9440 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16832/
   [junit4]   2> 1524392 T9440 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16800/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderInitiatedRecoveryOnCommitTest -Dtests.method=test -Dtests.seed=8632F28962AA5D4B -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=fi -Dtests.timezone=America/Argentina/Cordoba -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   79.1s J2 | LeaderInitiatedRecoveryOnCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:16808/c8n_1x3_commits_shard1_replica1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([8632F28962AA5D4B:E66CD53CC5630B3]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:16808 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 48 more
   [junit4]   2> 1524405 T9440 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8632F28962AA5D4B-001
   [junit4]   2> 79140 T9439 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=DefaultSimilarity, locale=fi, timezone=America/Argentina/Cordoba
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=181910080,total=352321536
   [junit4]   2> NOTE: All tests run in this JVM: [TestSolrQueryParserResource, ChangedSchemaMergeTest, BinaryUpdateRequestHandlerTest, RecoveryZkTest, UnloadDistributedZkTest, HdfsWriteToMultipleCollectionsTest, TermVectorComponentDistributedTest, ExternalFileFieldSortTest, SolrInfoMBeanTest, TestLRUCache, TestCopyFieldCollectionResource, XsltUpdateRequestHandlerTest, DistributedQueueTest, ScriptEngineTest, CachingDirectoryFactoryTest, TestReplicationHandler, PreAnalyzedUpdateProcessorTest, FullSolrCloudDistribCmdsTest, SchemaVersionSpecificBehaviorTest, ActionThrottleTest, TestRestManager, TestFastOutputStream, SharedFSAutoReplicaFailoverUtilsTest, SuggestComponentTest, QueryEqualityTest, TestSolrQueryParser, LeaderElectionIntegrationTest, DistributedDebugComponentTest, TestSimpleTrackingShardHandler, BJQParserTest, DistributedQueryComponentCustomSortTest, TestShardHandlerFactory, TestFaceting, TestSolrDeletionPolicy1, PolyFieldTest, AliasIntegrationTest, JSONWriterTest, HighlighterConfigTest, TestMacros, TestCollationField, PrimUtilsTest, SolrCoreCheckLockOnStartupTest, DistanceUnitsTest, SpellingQueryConverterTest, TestRequestStatusCollectionAPI, TestRangeQuery, TestStressReorder, TestInitParams, DistanceFunctionTest, TestOmitPositions, TestCryptoKeys, SyncSliceTest, CacheHeaderTest, MigrateRouteKeyTest, TestDynamicLoading, TestReload, SolrCmdDistributorTest, TestCSVResponseWriter, HdfsBasicDistributedZkTest, TestSolrConfigHandlerCloud, LeaderInitiatedRecoveryOnCommitTest]
   [junit4] Completed on J2 in 79.93s, 1 test, 1 error <<< FAILURES!

[...truncated 742 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 53 minutes 54 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 214079855 bytes
Compression is 0.0%
Took 41 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2884 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2884/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:20844/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:20844/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([691015E6B7971E47:E1442A3C196B73BF]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:20844 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:21221/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:21221/collection1
	at __randomizedtesting.SeedInfo.seed([691015E6B7971E47:E1442A3C196B73BF]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:21221/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:21221 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:21032/pv/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:21032/pv/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([691015E6B7971E47:E1442A3C196B73BF]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:21032/pv/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:21032 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 10033 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/init-core-data-001
   [junit4]   2> 1450379 T4115 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 1450384 T4115 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1450384 T4116 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1450484 T4115 oasc.ZkTestServer.run start zk server on port:20824
   [junit4]   2> 1450485 T4115 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1450485 T4115 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1450488 T4123 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@73910be7 name:ZooKeeperConnection Watcher:127.0.0.1:20824 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1450489 T4115 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1450489 T4115 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1450489 T4115 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1450492 T4115 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1450492 T4115 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1450493 T4126 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@70fb2281 name:ZooKeeperConnection Watcher:127.0.0.1:20824/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1450493 T4115 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1450494 T4115 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1450494 T4115 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1450495 T4115 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1450496 T4115 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1450497 T4115 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1450499 T4115 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1450499 T4115 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1450501 T4115 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1450502 T4115 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1450604 T4115 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1450604 T4115 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1450606 T4115 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1450606 T4115 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1450607 T4115 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1450608 T4115 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1450609 T4115 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1450610 T4115 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1450611 T4115 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1450611 T4115 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1450613 T4115 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1450613 T4115 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1450615 T4115 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1450615 T4115 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1450616 T4115 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1450617 T4115 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1450618 T4115 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1450619 T4115 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1452445 T4115 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/control-001/cores/collection1
   [junit4]   2> 1452450 T4115 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1452452 T4115 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:20834
   [junit4]   2> 1452453 T4115 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/control-001/cores, hostContext=/, hostPort=20833, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/tempDir-001/control/data}
   [junit4]   2> 1452453 T4115 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 1452453 T4115 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/control-001/'
   [junit4]   2> 1452496 T4115 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/control-001/solr.xml
   [junit4]   2> 1452549 T4115 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/control-001/cores
   [junit4]   2> 1452550 T4115 oasc.CoreContainer.<init> New CoreContainer 1479276749
   [junit4]   2> 1452550 T4115 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/control-001/]
   [junit4]   2> 1452550 T4115 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/control-001/lib
   [junit4]   2> 1452551 T4115 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/control-001/lib).
   [junit4]   2> 1452560 T4115 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1452560 T4115 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1452561 T4115 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1452561 T4115 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1452561 T4115 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1452561 T4115 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1452561 T4115 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1452562 T4115 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1452562 T4115 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1452562 T4115 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1452562 T4115 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1452569 T4115 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1452570 T4115 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1452570 T4115 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1452571 T4115 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1452571 T4115 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:20824/solr
   [junit4]   2> 1452571 T4115 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1452571 T4115 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1452572 T4115 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1452573 T4138 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@12323214 name:ZooKeeperConnection Watcher:127.0.0.1:20824 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1452574 T4115 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1452575 T4115 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1452576 T4115 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1452577 T4141 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@52569ce1 name:ZooKeeperConnection Watcher:127.0.0.1:20824/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1452578 T4115 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1452579 T4115 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1452580 T4115 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1452581 T4115 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1452582 T4115 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1452583 T4115 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1452585 T4115 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1452586 T4115 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1452587 T4115 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1452587 T4115 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:20833_
   [junit4]   2> 1452588 T4115 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:20833_
   [junit4]   2> 1452589 T4115 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1452590 T4115 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1452590 T4115 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1452591 T4115 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:20833_
   [junit4]   2> 1452592 T4115 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1452592 T4115 oasc.Overseer.start Overseer (id=93590608661118979-127.0.0.1:20833_-n_0000000000) starting
   [junit4]   2> 1452594 T4115 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1452597 T4115 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 1452598 T4143 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1452598 T4115 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1452598 T4142 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1452601 T4115 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/control-001/cores
   [junit4]   2> 1452603 T4115 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {shard=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/control-001/cores/collection1, schema=schema.xml, coreNodeName=, collection=control_collection, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/control-001/cores/collection1/, config=solrconfig.xml, loadOnStartup=true, name=collection1, dataDir=data/, transient=false}
   [junit4]   2> 1452603 T4115 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/control-001/cores/collection1/
   [junit4]   2> 1452603 T4115 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1452604 T4145 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1452604 T4145 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1452605 T4141 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1452605 T4145 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1452605 T4142 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20833",
   [junit4]   2> 	  "node_name":"127.0.0.1:20833_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 0
   [junit4]   2> 1452605 T4142 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20833",
   [junit4]   2> 	  "node_name":"127.0.0.1:20833_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection"}
   [junit4]   2> 1452606 T4142 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 1452606 T4142 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1452607 T4141 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1453605 T4145 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1453606 T4145 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1453606 T4145 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1453607 T4145 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1453607 T4145 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1453607 T4145 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/control-001/cores/collection1/'
   [junit4]   2> 1453626 T4145 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1453628 T4145 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1453639 T4145 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1453662 T4145 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1453712 T4145 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1453814 T4145 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1453826 T4145 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1454262 T4145 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1454279 T4145 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1454282 T4145 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1454290 T4145 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1454295 T4145 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1454299 T4145 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1454299 T4145 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1454299 T4145 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1454300 T4145 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1454300 T4145 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1454300 T4145 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1454301 T4145 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1454301 T4145 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1454301 T4145 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/control-001/cores/collection1/data/
   [junit4]   2> 1454302 T4145 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5fa863b5
   [junit4]   2> 1454302 T4145 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/control-001/cores/collection1/data
   [junit4]   2> 1454303 T4145 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/control-001/cores/collection1/data/index/
   [junit4]   2> 1454303 T4145 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1454303 T4145 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/control-001/cores/collection1/data/index
   [junit4]   2> 1454304 T4145 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=10, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.15179102665711916]
   [junit4]   2> 1454305 T4145 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7188116d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1c304556),segFN=segments_1,generation=1}
   [junit4]   2> 1454305 T4145 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1454310 T4145 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1454311 T4145 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1454311 T4145 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1454311 T4145 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1454311 T4145 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1454312 T4145 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1454312 T4145 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1454312 T4145 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1454313 T4145 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1454336 T4145 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1454338 T4145 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1454341 T4145 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1454343 T4145 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1454348 T4145 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/system,/admin/segments,/admin/file,/admin/mbeans,standard,/admin/properties,/admin/threads,/update/json/docs,/admin/luke,/get,/config,/update/csv,/update/json,/admin/ping,/update,/schema,/replication,/admin/logging,/admin/plugins
   [junit4]   2> 1454350 T4145 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1454352 T4145 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1454352 T4145 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1454354 T4145 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1454354 T4145 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1454355 T4145 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=50, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5958199606090643]
   [junit4]   2> 1454355 T4145 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7188116d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1c304556),segFN=segments_1,generation=1}
   [junit4]   2> 1454355 T4145 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1454356 T4145 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@27aa81d8[collection1] main
   [junit4]   2> 1454356 T4145 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1454357 T4145 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1454357 T4145 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1454357 T4145 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1454358 T4145 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1454358 T4145 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1454358 T4145 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1454359 T4145 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1454359 T4145 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1454359 T4145 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1454360 T4145 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1454360 T4146 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@27aa81d8[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1454360 T4149 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:20833 collection:control_collection shard:shard1
   [junit4]   2> 1454361 T4115 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 1454361 T4115 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1454361 T4149 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1454364 T4149 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1454365 T4141 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1454365 T4115 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1454365 T4149 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1454365 T4149 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 1454366 T4142 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C2151 name=collection1 org.apache.solr.core.SolrCore@1a3681ad url=http://127.0.0.1:20833/collection1 node=127.0.0.1:20833_ C2151_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:20833, node_name=127.0.0.1:20833_, core=collection1}
   [junit4]   2> 1454366 T4149 C:control_collection S:shard1 C2151 P20833 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:20833/collection1/
   [junit4]   2> 1454366 T4115 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1454366 T4149 C:control_collection S:shard1 C2151 P20833 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1454366 T4149 C:control_collection S:shard1 C2151 P20833 oasc.SyncStrategy.syncToMe http://127.0.0.1:20833/collection1/ has no replicas
   [junit4]   2> 1454367 T4149 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:20833/collection1/ shard1
   [junit4]   2> 1454367 T4149 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1454367 T4141 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1454367 T4153 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@ef73097 name:ZooKeeperConnection Watcher:127.0.0.1:20824/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1454367 T4115 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1454368 T4115 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1454368 T4115 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1454369 T4141 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1454370 T4142 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20833",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 1454370 T4115 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1454472 T4141 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1454472 T4153 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1454522 T4149 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:20833/collection1/ and leader is http://127.0.0.1:20833/collection1/
   [junit4]   2> 1454523 T4149 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:20833
   [junit4]   2> 1454523 T4149 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1454523 T4149 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1454523 T4149 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1454524 T4141 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1454525 T4142 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20833",
   [junit4]   2> 	  "node_name":"127.0.0.1:20833_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 3
   [junit4]   2> 1454526 T4142 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20833",
   [junit4]   2> 	  "node_name":"127.0.0.1:20833_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection"}
   [junit4]   2> 1454628 T4153 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1454628 T4141 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1456038 T4115 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001/cores/collection1
   [junit4]   2> 1456042 T4115 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001
   [junit4]   2> 1456043 T4115 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1456046 T4115 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:20845
   [junit4]   2> 1456046 T4115 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solrconfig=solrconfig.xml, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001/cores, hostPort=20844, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/tempDir-001/jetty1, hostContext=/}
   [junit4]   2> 1456047 T4115 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 1456047 T4115 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001/'
   [junit4]   2> 1456096 T4115 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001/solr.xml
   [junit4]   2> 1456155 T4115 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001/cores
   [junit4]   2> 1456156 T4115 oasc.CoreContainer.<init> New CoreContainer 1782224232
   [junit4]   2> 1456156 T4115 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001/]
   [junit4]   2> 1456156 T4115 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001/lib
   [junit4]   2> 1456157 T4115 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001/lib).
   [junit4]   2> 1456171 T4115 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1456172 T4115 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1456172 T4115 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1456172 T4115 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1456173 T4115 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1456173 T4115 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1456173 T4115 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1456173 T4115 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1456174 T4115 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1456174 T4115 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1456174 T4115 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1456182 T4115 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1456183 T4115 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1456183 T4115 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1456184 T4115 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1456184 T4115 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:20824/solr
   [junit4]   2> 1456184 T4115 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1456185 T4115 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1456186 T4115 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1456187 T4165 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7e6580cf name:ZooKeeperConnection Watcher:127.0.0.1:20824 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1456188 T4115 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1456189 T4115 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1456191 T4115 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1456192 T4168 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@27f768a name:ZooKeeperConnection Watcher:127.0.0.1:20824/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1456192 T4115 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1456196 T4115 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1457200 T4115 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:20844_
   [junit4]   2> 1457200 T4115 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:20844_
   [junit4]   2> 1457203 T4115 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1457205 T4115 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001/cores
   [junit4]   2> 1457207 T4115 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {loadOnStartup=true, dataDir=data/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001/cores/collection1, transient=false, shard=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001/cores/collection1/, schema=schema.xml, config=solrconfig.xml, coreNodeName=, collection=collection1, name=collection1}
   [junit4]   2> 1457208 T4115 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001/cores/collection1/
   [junit4]   2> 1457208 T4115 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1457209 T4169 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1457209 T4169 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1457210 T4169 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1457210 T4141 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1457211 T4142 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20844",
   [junit4]   2> 	  "node_name":"127.0.0.1:20844_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1"} current state version: 4
   [junit4]   2> 1457211 T4142 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20844",
   [junit4]   2> 	  "node_name":"127.0.0.1:20844_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1"}
   [junit4]   2> 1457211 T4142 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 1457211 T4142 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1457212 T4141 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1457212 T4168 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1457212 T4153 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1458210 T4169 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1458211 T4169 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1458211 T4169 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1458211 T4169 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1458212 T4169 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1458212 T4169 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001/cores/collection1/'
   [junit4]   2> 1458241 T4169 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1458244 T4169 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1458255 T4169 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1458274 T4169 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1458314 T4169 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1458415 T4169 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1458423 T4169 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1458847 T4169 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1458860 T4169 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1458862 T4169 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1458869 T4169 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1458873 T4169 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1458876 T4169 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1458876 T4169 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1458877 T4169 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1458878 T4169 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1458878 T4169 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1458878 T4169 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1458878 T4169 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1458879 T4169 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1458879 T4169 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 1458879 T4169 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5fa863b5
   [junit4]   2> 1458880 T4169 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001/cores/collection1/data
   [junit4]   2> 1458880 T4169 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 1458880 T4169 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1458880 T4169 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 691015E6B7971E47-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 1458881 T4169 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=10, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.15179102665711916]
   [junit4]   2> 1458881 T4169 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@76847976 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73426ec3),segFN=segments_1,generation=1}
   [junit4]   2> 1458881 T4169 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1458886 T4169 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1458886 T4169 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1458887 T4169 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1458887 T4169 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1458887 T4169 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1458887 T4169 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1458888 T4169 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1458888 T4169 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1458888 T4169 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1458902 T4169 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1458904 T4169 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1458905 T4169 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1458907 T4169 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1458912 T4169 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/system,/admin/segments,/admin/file,/admin/mbeans,standard,/admin/properties,/admin/threads,/update/json/docs,/admin/luke,/get,/config,/update/csv,/update/json,/admin/ping,/update,/schema,/replication,/admin/logging,/admin/plugins
   [junit4]   2> 1458914 T4169 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1458915 T4169 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1458915 T4169 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1458916 T4169 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1458916 T4169 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1458917 T4169 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=50, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5958199606090643]
   [junit4]   2> 1458918 T4169 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@76847976 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@73426ec3),segFN=segments_1,generation=1}
   [junit4]   2> 1458918 T4169 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1458918 T4169 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@1d87e561[collection1] main
   [junit4]   2> 1458918 T4169 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1458919 T4169 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1458919 T4169 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1458919 T4169 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1458920 T4169 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1458920 T4169 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1458920 T4169 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1458920 T4169 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1458920 T4169 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1458921 T4169 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1458922 T4170 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1d87e561[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1458922 T4169 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1458923 T4173 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:20844 collection:collection1 shard:shard1
   [junit4]   2> 1458923 T4115 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 1458923 T4115 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1458924 T4173 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1458927 T4173 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1458928 T4141 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1458928 T4173 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1458929 T4173 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C2152 name=collection1 org.apache.solr.core.SolrCore@4d26573b url=http://127.0.0.1:20844/collection1 node=127.0.0.1:20844_ C2152_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:20844, node_name=127.0.0.1:20844_, core=collection1}
   [junit4]   2> 1458929 T4173 C:collection1 S:shard1 C2152 P20844 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:20844/collection1/
   [junit4]   2> 1458929 T4142 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 1458929 T4173 C:collection1 S:shard1 C2152 P20844 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1458930 T4173 C:collection1 S:shard1 C2152 P20844 oasc.SyncStrategy.syncToMe http://127.0.0.1:20844/collection1/ has no replicas
   [junit4]   2> 1458930 T4173 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:20844/collection1/ shard1
   [junit4]   2> 1458930 T4173 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1458930 T4168 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1458930 T4153 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1458930 T4141 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1458933 T4141 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1458934 T4142 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20844",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 1459036 T4141 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1459036 T4153 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1459036 T4168 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1459086 T4173 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:20844/collection1/ and leader is http://127.0.0.1:20844/collection1/
   [junit4]   2> 1459087 T4173 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:20844
   [junit4]   2> 1459087 T4173 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1459087 T4173 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 1459087 T4173 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1459088 T4141 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1459089 T4142 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20844",
   [junit4]   2> 	  "node_name":"127.0.0.1:20844_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1"} current state version: 7
   [junit4]   2> 1459089 T4142 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20844",
   [ju

[...truncated too long message...]

]   2> 1611043 T6476 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:21210 21210
   [junit4]   2> 1611246 T6477 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:21210 21210
   [junit4]   2> 1611247 T6477 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1611247 T6476 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:21221/
   [junit4]   2> 1611248 T6476 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:21213/
   [junit4]   2> 1611248 T6476 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:21229/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=691015E6B7971E47 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=nl_NL -Dtests.timezone=America/St_Lucia -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   67.5s J2 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:21221/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([691015E6B7971E47:E1442A3C196B73BF]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:21221/collection1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 49 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:21221 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 54 more
   [junit4]   2> 1611287 T6476 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 691015E6B7971E47-001
   [junit4]   2> NOTE: test params are: codec=FastDecompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=4775, maxDocsPerChunk=4, blockSize=57), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=4775, blockSize=57)), sim=RandomSimilarityProvider(queryNorm=false,coord=yes): {}, locale=nl_NL, timezone=America/St_Lucia
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=183747848,total=338165760
   [junit4]   2> NOTE: All tests run in this JVM: [DistributedIntervalFacetingTest, SpellCheckCollatorTest, MultiTermTest, PreAnalyzedFieldTest, SpatialHeatmapFacetsTest, ConnectionManagerTest, RAMDirectoryFactoryTest, TestSolrXml, TestRecovery, TestWriterPerf, TestSolrDeletionPolicy1, SOLR749Test, TestConfigOverlay, TestDynamicFieldCollectionResource, TestSearchPerf, DefaultValueUpdateProcessorTest, TestCoreContainer, UUIDFieldTest, DeleteReplicaTest, BJQParserTest, TestSolrIndexConfig, TestFieldResource, ClusterStateUpdateTest, TestSort, SuggesterTSTTest, TestClusterStateMutator, SolrCoreTest, TestJoin, CachingDirectoryFactoryTest, UniqFieldsUpdateProcessorFactoryTest, SolrIndexSplitterTest, TestSearcherReuse, SimpleFacetsTest, AtomicUpdatesTest, TestDistributedSearch, FieldAnalysisRequestHandlerTest, SolrPluginUtilsTest, SSLMigrationTest, TestDFRSimilarityFactory, TestLRUCache, RankQueryTest, AddBlockUpdateTest, DocValuesMissingTest, XmlUpdateRequestHandlerTest, RollingRestartTest, TestCodecSupport, QueryResultKeyTest, SharedFSAutoReplicaFailoverTest, TestExpandComponent, TestIntervalFaceting, DistributedMLTComponentTest, TestConfigSets, FieldMutatingUpdateProcessorTest, CoreAdminRequestStatusTest, DistributedDebugComponentTest, DistributedSpellCheckComponentTest, TestComponentsName, TestRebalanceLeaders, LeaderFailoverAfterPartitionTest, BasicDistributedZkTest, ChaosMonkeyNothingIsSafeTest, TestSolrConfigHandlerCloud, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J2 in 67.50s, 1 test, 1 error <<< FAILURES!

[...truncated 690 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 53 minutes 25 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 218077853 bytes
Compression is 0.0%
Took 38 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2883 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2883/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:49400/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:49400/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([613D1676A5A50840:E96929AC0B5965B8]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([613D1676A5A50840:E96929AC0B5965B8]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([613D1676A5A50840:E96929AC0B5965B8]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9361 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/init-core-data-001
   [junit4]   2> 410808 T986 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /b/tv
   [junit4]   2> 410816 T986 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 410817 T987 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 410917 T986 oasc.ZkTestServer.run start zk server on port:11040
   [junit4]   2> 410918 T986 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 410919 T986 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 410923 T994 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4b310ac6 name:ZooKeeperConnection Watcher:127.0.0.1:11040 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 410924 T986 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 410924 T986 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 410925 T986 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 410929 T986 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 410930 T986 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 410931 T997 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4cca2544 name:ZooKeeperConnection Watcher:127.0.0.1:11040/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 410932 T986 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 410932 T986 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 410933 T986 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 410935 T986 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 410937 T986 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 410938 T986 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 410940 T986 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 410940 T986 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 410943 T986 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 410944 T986 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 410946 T986 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 410947 T986 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 410949 T986 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 410949 T986 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 410951 T986 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 410951 T986 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 410953 T986 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 410953 T986 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 410955 T986 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 410956 T986 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 410957 T986 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 410958 T986 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 410960 T986 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 410960 T986 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 410962 T986 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 410963 T986 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 410964 T986 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 410965 T986 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 412310 T986 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/control-001/cores/collection1
   [junit4]   2> 412315 T986 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 412320 T986 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:11079
   [junit4]   2> 412320 T986 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/b/tv, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/tempDir-001/control/data, hostPort=11078, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/control-001/cores}
   [junit4]   2> 412321 T986 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3b061299
   [junit4]   2> 412321 T986 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/control-001/'
   [junit4]   2> 412360 T986 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/control-001/solr.xml
   [junit4]   2> 412406 T986 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/control-001/cores
   [junit4]   2> 412407 T986 oasc.CoreContainer.<init> New CoreContainer 1556104529
   [junit4]   2> 412407 T986 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/control-001/]
   [junit4]   2> 412407 T986 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/control-001/lib
   [junit4]   2> 412408 T986 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/control-001/lib).
   [junit4]   2> 412421 T986 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 412421 T986 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 412421 T986 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 412422 T986 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 412422 T986 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 412422 T986 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 412422 T986 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 412423 T986 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 412423 T986 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 412423 T986 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 412424 T986 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 412424 T986 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 412425 T986 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 412425 T986 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 412425 T986 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 412425 T986 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:11040/solr
   [junit4]   2> 412426 T986 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 412426 T986 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 412427 T986 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 412429 T1009 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4eb0ad7c name:ZooKeeperConnection Watcher:127.0.0.1:11040 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 412430 T986 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 412430 T986 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 412432 T986 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 412434 T1012 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5c731511 name:ZooKeeperConnection Watcher:127.0.0.1:11040/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 412434 T986 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 412435 T986 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 412437 T986 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 412439 T986 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 412441 T986 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 412443 T986 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 412445 T986 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 412446 T986 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 412447 T986 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 412448 T986 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:11078_b%2Ftv
   [junit4]   2> 412449 T986 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:11078_b%2Ftv
   [junit4]   2> 412450 T986 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 412452 T986 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 412453 T986 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 412454 T986 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:11078_b%2Ftv
   [junit4]   2> 412455 T986 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 412456 T986 oasc.Overseer.start Overseer (id=93589584465756163-127.0.0.1:11078_b%2Ftv-n_0000000000) starting
   [junit4]   2> 412458 T986 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 412463 T986 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 412464 T986 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 412464 T1013 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 412464 T1014 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 412469 T986 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/control-001/cores
   [junit4]   2> 412471 T986 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {shard=, collection=control_collection, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/control-001/cores/collection1, dataDir=data/, coreNodeName=, transient=false, loadOnStartup=true, schema=schema.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/control-001/cores/collection1/, name=collection1, config=solrconfig.xml}
   [junit4]   2> 412472 T986 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/control-001/cores/collection1/
   [junit4]   2> 412472 T986 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 412474 T1016 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 412474 T1016 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 412475 T1016 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 412475 T1012 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 412476 T1013 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:11078_b%2Ftv",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11078/b/tv"} current state version: 0
   [junit4]   2> 412477 T1013 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:11078_b%2Ftv",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11078/b/tv"}
   [junit4]   2> 412477 T1013 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 412477 T1013 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 412479 T1012 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 413475 T1016 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 413476 T1016 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 413476 T1016 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 413476 T1016 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 413477 T1016 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 413477 T1016 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/control-001/cores/collection1/'
   [junit4]   2> 413490 T1016 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 413492 T1016 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 413501 T1016 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 413517 T1016 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 413552 T1016 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 413553 T1016 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 413561 T1016 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 413848 T1016 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 413860 T1016 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 413862 T1016 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 413871 T1016 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 413875 T1016 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 413878 T1016 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 413878 T1016 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 413878 T1016 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 413879 T1016 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 413879 T1016 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 413879 T1016 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 413880 T1016 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 413880 T1016 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 413880 T1016 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/control-001/cores/collection1/data/
   [junit4]   2> 413880 T1016 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@592b94df
   [junit4]   2> 413881 T1016 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/control-001/cores/collection1/data
   [junit4]   2> 413881 T1016 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/control-001/cores/collection1/data/index/
   [junit4]   2> 413881 T1016 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 413882 T1016 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/control-001/cores/collection1/data/index
   [junit4]   2> 413882 T1016 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=32, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 413883 T1016 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@dd6ce4 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4917ed2d),segFN=segments_1,generation=1}
   [junit4]   2> 413883 T1016 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 413888 T1016 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 413888 T1016 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 413888 T1016 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 413889 T1016 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 413889 T1016 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 413889 T1016 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 413889 T1016 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 413890 T1016 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 413890 T1016 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 413902 T1016 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 413904 T1016 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 413906 T1016 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 413907 T1016 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 413912 T1016 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/luke,/schema,/config,/update,/update/csv,/update/json,/replication,standard,/admin/segments,/admin/ping,/admin/plugins,/admin/file,/update/json/docs,/admin/logging,/admin/mbeans,/admin/threads,/admin/system,/admin/properties,/get
   [junit4]   2> 413913 T1016 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 413914 T1016 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 413915 T1016 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 413916 T1016 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 413916 T1016 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 413916 T1016 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=33, maxMergeAtOnceExplicit=13, maxMergedSegmentMB=17.318359375, floorSegmentMB=0.9638671875, forceMergeDeletesPctAllowed=8.785740902982491, segmentsPerTier=28.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3474586037465189
   [junit4]   2> 413917 T1016 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@dd6ce4 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4917ed2d),segFN=segments_1,generation=1}
   [junit4]   2> 413917 T1016 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 413917 T1016 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@733b8bc1[collection1] main
   [junit4]   2> 413918 T1016 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 413918 T1016 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 413918 T1016 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 413919 T1016 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 413919 T1016 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 413919 T1016 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 413919 T1016 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 413920 T1016 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 413920 T1016 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 413920 T1016 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 413921 T1017 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@733b8bc1[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 413921 T1016 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 413922 T1020 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:11078/b/tv collection:control_collection shard:shard1
   [junit4]   2> 413922 T986 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 413923 T986 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 413923 T1020 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 413925 T986 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 413926 T986 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 413927 T1020 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 413927 T1024 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5fc5dbc7 name:ZooKeeperConnection Watcher:127.0.0.1:11040/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 413928 T986 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 413928 T1012 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 413928 T1020 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 413928 T986 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 413928 T1020 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 413928 T1013 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 413928 T986 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> ASYNC  NEW_CORE C257 name=collection1 org.apache.solr.core.SolrCore@a455eab url=http://127.0.0.1:11078/b/tv/collection1 node=127.0.0.1:11078_b%2Ftv C257_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:11078_b%2Ftv, state=down, core=collection1, base_url=http://127.0.0.1:11078/b/tv}
   [junit4]   2> 413929 T1020 C:control_collection S:shard1 C257 P11078 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:11078/b/tv/collection1/
   [junit4]   2> 413929 T1020 C:control_collection S:shard1 C257 P11078 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 413929 T1020 C:control_collection S:shard1 C257 P11078 oasc.SyncStrategy.syncToMe http://127.0.0.1:11078/b/tv/collection1/ has no replicas
   [junit4]   2> 413929 T1020 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:11078/b/tv/collection1/ shard1
   [junit4]   2> 413930 T1020 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 413931 T986 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 413932 T1012 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 413933 T1013 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11078/b/tv",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 1
   [junit4]   2> 414035 T1024 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 414035 T1012 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 414085 T1020 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:11078/b/tv/collection1/ and leader is http://127.0.0.1:11078/b/tv/collection1/
   [junit4]   2> 414086 T1020 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:11078/b/tv
   [junit4]   2> 414086 T1020 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 414086 T1020 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 414086 T1020 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 414087 T1012 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 414088 T1013 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:11078_b%2Ftv",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11078/b/tv"} current state version: 2
   [junit4]   2> 414088 T1013 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:11078_b%2Ftv",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11078/b/tv"}
   [junit4]   2> 414191 T1012 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 414191 T1024 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 414895 T986 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001/cores/collection1
   [junit4]   2> 414897 T986 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001
   [junit4]   2> 414898 T986 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 414901 T986 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:11085
   [junit4]   2> 414901 T986 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/tempDir-001/jetty1, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001/cores, hostContext=/b/tv, hostPort=11084, solrconfig=solrconfig.xml}
   [junit4]   2> 414902 T986 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3b061299
   [junit4]   2> 414902 T986 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001/'
   [junit4]   2> 414935 T986 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001/solr.xml
   [junit4]   2> 414972 T986 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001/cores
   [junit4]   2> 414973 T986 oasc.CoreContainer.<init> New CoreContainer 2063040257
   [junit4]   2> 414973 T986 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001/]
   [junit4]   2> 414973 T986 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001/lib
   [junit4]   2> 414974 T986 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001/lib).
   [junit4]   2> 414984 T986 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 414984 T986 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 414984 T986 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 414985 T986 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 414985 T986 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 414985 T986 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 414985 T986 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 414985 T986 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 414986 T986 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 414986 T986 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 414986 T986 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 414987 T986 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 414987 T986 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 414987 T986 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 414987 T986 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 414988 T986 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:11040/solr
   [junit4]   2> 414988 T986 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 414988 T986 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 414989 T986 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 414990 T1036 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@73f56cc0 name:ZooKeeperConnection Watcher:127.0.0.1:11040 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 414991 T986 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 414992 T986 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 414994 T986 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 414996 T1039 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@11cd7ee0 name:ZooKeeperConnection Watcher:127.0.0.1:11040/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 414996 T986 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 414999 T986 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 416003 T986 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:11084_b%2Ftv
   [junit4]   2> 416004 T986 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:11084_b%2Ftv
   [junit4]   2> 416007 T986 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 416009 T986 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001/cores
   [junit4]   2> 416011 T986 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, transient=false, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001/cores/collection1, config=solrconfig.xml, collection=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001/cores/collection1/, dataDir=data/, coreNodeName=, shard=, loadOnStartup=true}
   [junit4]   2> 416011 T986 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001/cores/collection1/
   [junit4]   2> 416012 T986 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 416013 T1040 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 416014 T1040 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 416014 T1012 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 416014 T1040 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 416015 T1013 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:11084_b%2Ftv",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11084/b/tv"} current state version: 3
   [junit4]   2> 416016 T1013 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:11084_b%2Ftv",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11084/b/tv"}
   [junit4]   2> 416016 T1013 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 416016 T1013 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 416017 T1039 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 416017 T1024 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 416017 T1012 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 417015 T1040 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 417016 T1040 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 417016 T1040 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 417016 T1040 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 417017 T1040 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 417017 T1040 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001/cores/collection1/'
   [junit4]   2> 417030 T1040 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 417032 T1040 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 417041 T1040 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 417056 T1040 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 417091 T1040 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 417092 T1040 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 417100 T1040 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 417378 T1040 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 417397 T1040 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 417399 T1040 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 417408 T1040 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 417412 T1040 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 417415 T1040 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 417416 T1040 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 417416 T1040 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 417416 T1040 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 417417 T1040 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 417417 T1040 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 417417 T1040 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 417417 T1040 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 417418 T1040 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 417418 T1040 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@592b94df
   [junit4]   2> 417419 T1040 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001/cores/collection1/data
   [junit4]   2> 417419 T1040 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 417419 T1040 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 417419 T1040 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 417420 T1040 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=32, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 417420 T1040 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6ae174eb lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@77c3271e),segFN=segments_1,generation=1}
   [junit4]   2> 417420 T1040 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 417426 T1040 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 417427 T1040 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 417427 T1040 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 417427 T1040 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 417427 T1040 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 417427 T1040 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 417428 T1040 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 417428 T1040 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 417428 T1040 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 417441 T1040 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 417443 T1040 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 417444 T1040 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 417446 T1040 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 417452 T1040 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/luke,/schema,/config,/update,/update/csv,/update/json,/replication,standard,/admin/segments,/admin/ping,/admin/plugins,/admin/file,/update/json/docs,/admin/logging,/admin/mbeans,/admin/threads,/admin/system,/admin/properties,/get
   [junit4]   2> 417454 T1040 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 417455 T1040 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 417455 T1040 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 417456 T1040 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 417456 T1040 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 417457 T1040 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=33, maxMergeAtOnceExplicit=13, maxMergedSegmentMB=17.318359375, floorSegmentMB=0.9638671875, forceMergeDeletesPctAllowed=8.785740902982491, segmentsPerTier=28.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3474586037465189
   [junit4]   2> 417458 T1040 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6ae174eb lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@77c3271e),segFN=segments_1,generation=1}
   [junit4]   2> 417458 T1040 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 417458 T1040 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@32f78197[collection1] main
   [junit4]   2> 417458 T1040 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 417459 T1040 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 417459 T1040 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 417459 T1040 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 417460 T1040 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 417460 T1040 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 417460 T1040 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 417460 T1040 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 417461 T1040 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 417461 T1040 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 417462 T1041 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@32f78197[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 417462 T1040 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 417463 T1044 C:collection1 S:shard2 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:11084/b/tv collection:collection1 shard:shard2
   [junit4]   2> 417463 T986 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 417464 T986 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 417464 T1044 C:collection1 S:shard2 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 417467 T1044 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 417469 T1012 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 417469 T1044 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 417469 T1044 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C258 name=collection1 org.apache.solr.core.SolrCore@61b40f8c url=http://127.0.0.1:11084/b/tv/collection1 node=127.0.0.1:11084_b%2Ftv C258_STATE=coll:collection1 core:collection1 props:{node_name=127.0.0.1:11084_b%2Ftv, state=down, core=collection1, base_url=http://127.0.0.1:11084/b/tv}
   [junit4]   2> 417469 T1044 C:collection1 S:shard2 C258 P11084 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:11084/b/tv/collection1/
   [junit4]   2> 417469 T1013 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1"} current state version: 4
   [junit4]   2> 417469 T1044 C:collection1 S:shard2 C258 P11084 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 417470 T1044 C:collection1 S:shard2 C258 P11084 oasc.SyncStrategy.syncToMe http://127.0.0.1:11084/b/tv/collection1/ has no replicas
   [junit4]   2> 417470 T1044 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:11084/b/tv/collection1/ shard2
   [junit4]   2> 417470 T1044 C:collection1 S:shard2 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 417473 T1012 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 417473 T1013 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11084/b/tv",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 4
   [junit4]   2> 417576 T1012 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 417576 T1039 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 417576 T1024 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 417625 T1044 C:collection1 S:shard2 oasc.ZkController.register We are http://127.0.0.1:11084/b/tv/collection1/ and leader is http://127.0.0.1:11084/b/tv/collection1/
   [junit4]   2> 417626 T1044 C:collection1 S:shard2 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:11084/b/tv
   [junit4]   2> 417626 T1044 C:collection1 S:shard2 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 417626 T1044 C:collection1 S:shard2 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 417626 T1044 C:collection1 S:shard2 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 417627 T1012 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 417628 T1013 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:11084_b%2Ftv",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11084/b/tv"} current state version: 5
   [junit4]   2> 417628 T1013 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:11084_b%2Ftv",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11084/b/tv"}
   [junit4]   2> 417731 T1039 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 417731 T1024 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 417731 T1012 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 418404 T986 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-2-001/cores/collection1
   [junit4]   2> 418406 T986 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-2-001
   [junit4]   2> 418407 T986 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 418409 T986 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:11089
   [junit4]   2> 418409 T986 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/shard-2-001/cores, hostContext=/b/tv, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 613D1676A5A50840-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostPort=11088}
   [junit4]   2> 418410 T986 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3b061299
   [junit4]   2> 418410 T986 oasc.SolrResourceLoader

[...truncated too long message...]

red, but Solr cannot talk to ZK
   [junit4]   2> 928792 T5179 C:control_collection S:shard1 c:collection1 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> 928955 T5179 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:32175 32175
   [junit4]   2> 929019 T5180 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:32175 32175
   [junit4]   2> 929021 T5180 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 929021 T5179 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:32179/
   [junit4]   2> 929021 T5179 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:32198/
   [junit4]   2> 929022 T5179 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 6 connections to: http://127.0.0.1:32188/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=613D1676A5A50840 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ar_LB -Dtests.timezone=Indian/Cocos -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   75.9s J1 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([613D1676A5A50840:E96929AC0B5965B8]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 929028 T5179 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 613D1676A5A50840-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {id=Lucene50(blocksize=128), a_t=PostingsFormat(name=Memory doPackFST= false), timestamp=PostingsFormat(name=Memory doPackFST= false), intDefault=PostingsFormat(name=Memory doPackFST= false), text=PostingsFormat(name=Direct), range_facet_l=PostingsFormat(name=Memory doPackFST= false), multiDefault=Lucene50(blocksize=128), _version_=PostingsFormat(name=Memory doPackFST= false)}, docValues:{timestamp=DocValuesFormat(name=Asserting)}, sim=RandomSimilarityProvider(queryNorm=false,coord=yes): {}, locale=ar_LB, timezone=Indian/Cocos
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=129014776,total=289406976
   [junit4]   2> NOTE: All tests run in this JVM: [TestSchemaNameResource, TestRTGBase, TestEmbeddedSolrServerConstructors, MultiTermTest, AssignTest, OverriddenZkACLAndCredentialsProvidersTest, TestSolrQueryParserDefaultOperatorResource, DistribDocExpirationUpdateProcessorTest, DistributedTermsComponentTest, DocValuesTest, HighlighterTest, ShardSplitTest, TestSort, ReplicationFactorTest, TestConfigOverlay, CloudExitableDirectoryReaderTest, TestCollationField, NumericFieldsTest, TestDynamicFieldCollectionResource, DistanceUnitsTest, TimeZoneUtilsTest, FieldMutatingUpdateProcessorTest, DirectSolrSpellCheckerTest, TestReplicaProperties, TestPhraseSuggestions, UniqFieldsUpdateProcessorFactoryTest, TestPerFieldSimilarity, TestSolrConfigHandlerCloud, SolrCoreTest, AnalyticsMergeStrategyTest, TestSolr4Spatial2, SolrCmdDistributorTest, TestHdfsUpdateLog, AnalysisErrorHandlingTest, DistributedQueryComponentCustomSortTest, TestRecovery, DistributedSpellCheckComponentTest, TestLFUCache, FastVectorHighlighterTest, BadComponentTest, TestSchemaVersionResource, SpellCheckCollatorTest, TestCursorMarkWithoutUniqueKey, TestReplicationHandlerBackup, TestDistributedGrouping, OutOfBoxZkACLAndCredentialsProvidersTest, TestLRUCache, TestSimpleQParserPlugin, AddSchemaFieldsUpdateProcessorFactoryTest, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J1 in 75.92s, 1 test, 1 error <<< FAILURES!

[...truncated 937 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 54 minutes 32 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 219492621 bytes
Compression is 0.0%
Took 42 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2882 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2882/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:16230/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:16230/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([46171878E3F1D9B8:CE4327A24D0DB440]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:23333/i_zk/cc/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:23333/i_zk/cc/collection1
	at __randomizedtesting.SeedInfo.seed([46171878E3F1D9B8:CE4327A24D0DB440]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:23333/i_zk/cc/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:56796/zk/g/repfacttest_c8n_1x3_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:56796/zk/g/repfacttest_c8n_1x3_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([46171878E3F1D9B8:CE4327A24D0DB440]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:56796/zk/g/repfacttest_c8n_1x3_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9405 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/init-core-data-001
   [junit4]   2> 119902 T346 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /i_zk/cc
   [junit4]   2> 119919 T346 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 119920 T347 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 120021 T346 oasc.ZkTestServer.run start zk server on port:23310
   [junit4]   2> 120022 T346 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 120023 T346 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 120028 T354 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7501dca0 name:ZooKeeperConnection Watcher:127.0.0.1:23310 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 120029 T346 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 120029 T346 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 120030 T346 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 120034 T346 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 120035 T346 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 120037 T357 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@643ff3b2 name:ZooKeeperConnection Watcher:127.0.0.1:23310/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 120037 T346 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 120038 T346 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 120038 T346 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 120042 T346 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 120046 T346 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 120048 T346 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 120052 T346 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 120053 T346 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 120058 T346 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 120059 T346 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 120063 T346 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 120064 T346 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 120067 T346 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 120068 T346 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 120072 T346 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 120073 T346 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 120076 T346 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 120077 T346 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 120080 T346 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 120081 T346 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 120084 T346 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 120085 T346 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 120088 T346 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 120089 T346 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 120093 T346 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 120094 T346 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 120097 T346 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 120098 T346 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 121832 T346 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/control-001/cores/collection1
   [junit4]   2> 121837 T346 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 121841 T346 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:23317
   [junit4]   2> 121841 T346 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/control-001/cores, hostContext=/i_zk/cc, hostPort=23316}
   [junit4]   2> 121842 T346 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 121842 T346 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/control-001/'
   [junit4]   2> 121885 T346 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/control-001/solr.xml
   [junit4]   2> 121935 T346 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/control-001/cores
   [junit4]   2> 121936 T346 oasc.CoreContainer.<init> New CoreContainer 1844476828
   [junit4]   2> 121936 T346 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/control-001/]
   [junit4]   2> 121937 T346 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/control-001/lib
   [junit4]   2> 121937 T346 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/control-001/lib).
   [junit4]   2> 121948 T346 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 121948 T346 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 121948 T346 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 121949 T346 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 121949 T346 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 121949 T346 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 121949 T346 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 121950 T346 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 121950 T346 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 121950 T346 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 121950 T346 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 121951 T346 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 121951 T346 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 121951 T346 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 121952 T346 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 121952 T346 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:23310/solr
   [junit4]   2> 121952 T346 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 121952 T346 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 121953 T346 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 121955 T369 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@262a33a1 name:ZooKeeperConnection Watcher:127.0.0.1:23310 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 121956 T346 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 121957 T346 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 121960 T346 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 121961 T372 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2f756f1 name:ZooKeeperConnection Watcher:127.0.0.1:23310/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 121961 T346 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 121963 T346 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 121966 T346 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 121969 T346 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 121972 T346 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 121975 T346 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 121978 T346 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 121980 T346 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 121982 T346 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 121983 T346 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:23316_i_zk%2Fcc
   [junit4]   2> 121984 T346 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:23316_i_zk%2Fcc
   [junit4]   2> 121987 T346 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 121989 T346 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 121991 T346 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 121993 T346 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:23316_i_zk%2Fcc
   [junit4]   2> 121993 T346 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 121995 T346 oasc.Overseer.start Overseer (id=93588620005015555-127.0.0.1:23316_i_zk%2Fcc-n_0000000000) starting
   [junit4]   2> 121998 T346 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 122007 T346 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 122008 T374 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 122008 T346 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 122009 T373 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 122014 T346 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/control-001/cores
   [junit4]   2> 122030 T346 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, loadOnStartup=true, collection=control_collection, coreNodeName=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/control-001/cores/collection1/, dataDir=data/, shard=, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/control-001/cores/collection1, schema=schema.xml, transient=false}
   [junit4]   2> 122031 T346 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/control-001/cores/collection1/
   [junit4]   2> 122031 T346 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 122033 T376 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 122033 T376 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 122043 T372 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 122043 T376 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 122051 T373 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:23316_i_zk%2Fcc",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23316/i_zk/cc"} current state version: 0
   [junit4]   2> 122099 T373 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:23316_i_zk%2Fcc",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23316/i_zk/cc"}
   [junit4]   2> 122107 T373 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 122140 T373 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 122160 T372 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 123045 T376 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 123045 T376 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 123046 T376 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 123046 T376 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 123047 T376 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 123048 T376 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/control-001/cores/collection1/'
   [junit4]   2> 123097 T376 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 123115 T376 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 123145 T376 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 123165 T376 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 123202 T376 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 123204 T376 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 123214 T376 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 123622 T376 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 123643 T376 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 123646 T376 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 123664 T376 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 123670 T376 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 123675 T376 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 123676 T376 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 123676 T376 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 123677 T376 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 123678 T376 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 123678 T376 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 123678 T376 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 123679 T376 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 123679 T376 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/control-001/cores/collection1/data/
   [junit4]   2> 123680 T376 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3f430afc
   [junit4]   2> 123681 T376 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/control-001/cores/collection1/data
   [junit4]   2> 123681 T376 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/control-001/cores/collection1/data/index/
   [junit4]   2> 123682 T376 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 123682 T376 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/control-001/cores/collection1/data/index
   [junit4]   2> 123683 T376 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=32, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.45963115368360286]
   [junit4]   2> 123684 T376 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@10b5b606 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@9c14c7c),segFN=segments_1,generation=1}
   [junit4]   2> 123684 T376 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 123689 T376 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 123777 T376 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 123778 T376 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 123778 T376 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 123778 T376 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 123778 T376 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 123779 T376 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 123779 T376 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 123779 T376 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 123806 T376 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 123809 T376 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 123811 T376 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 123813 T376 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 123818 T376 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/plugins,/admin/segments,/admin/mbeans,/admin/ping,/update/csv,/update/json,/get,/config,/admin/threads,standard,/replication,/admin/logging,/admin/properties,/update,/update/json/docs,/admin/file,/admin/luke,/admin/system,/schema
   [junit4]   2> 123819 T376 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 123821 T376 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 123821 T376 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 123823 T376 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 123823 T376 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 123824 T376 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=15, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6488594503462402]
   [junit4]   2> 123824 T376 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@10b5b606 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@9c14c7c),segFN=segments_1,generation=1}
   [junit4]   2> 123824 T376 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 123825 T376 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@36b83e64[collection1] main
   [junit4]   2> 123825 T376 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 123826 T376 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 123827 T376 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 123840 T376 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 123841 T376 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 123841 T376 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 123842 T376 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 123842 T376 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 123842 T376 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 123843 T376 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 123844 T377 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@36b83e64[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 123851 T376 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 123852 T380 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:23316/i_zk/cc collection:control_collection shard:shard1
   [junit4]   2> 123853 T346 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 123853 T346 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 123867 T346 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 123868 T346 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 123870 T384 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4e606c8b name:ZooKeeperConnection Watcher:127.0.0.1:23310/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 123871 T346 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 123871 T346 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 123871 T346 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 123878 T346 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 123879 T346 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 123879 T346 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 123880 T346 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 123882 T387 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4f02246c name:ZooKeeperConnection Watcher:127.0.0.1:23310/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 123882 T346 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 123882 T346 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 123885 T372 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 123886 T373 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 1
   [junit4]   2> 123887 T373 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 123889 T373 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 123891 T372 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 123891 T384 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 123903 T380 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 123911 T380 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 123914 T372 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 123914 T380 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 123915 T380 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 123915 T373 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 2
   [junit4]   2> ASYNC  NEW_CORE C38 name=collection1 org.apache.solr.core.SolrCore@5c0cc960 url=http://127.0.0.1:23316/i_zk/cc/collection1 node=127.0.0.1:23316_i_zk%2Fcc C38_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:23316_i_zk%2Fcc, state=down, core=collection1, base_url=http://127.0.0.1:23316/i_zk/cc}
   [junit4]   2> 123925 T380 C:control_collection S:shard1 C38 P23316 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:23316/i_zk/cc/collection1/
   [junit4]   2> 123926 T380 C:control_collection S:shard1 C38 P23316 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 123926 T380 C:control_collection S:shard1 C38 P23316 oasc.SyncStrategy.syncToMe http://127.0.0.1:23316/i_zk/cc/collection1/ has no replicas
   [junit4]   2> 123927 T380 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:23316/i_zk/cc/collection1/ shard1
   [junit4]   2> 123941 T380 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 123947 T372 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 123948 T373 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23316/i_zk/cc",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 124054 T384 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 124054 T372 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 124101 T380 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:23316/i_zk/cc/collection1/ and leader is http://127.0.0.1:23316/i_zk/cc/collection1/
   [junit4]   2> 124101 T380 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:23316/i_zk/cc
   [junit4]   2> 124101 T380 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 124102 T380 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 124102 T380 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 124104 T372 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 124105 T373 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:23316_i_zk%2Fcc",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23316/i_zk/cc"} current state version: 3
   [junit4]   2> 124106 T373 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:23316_i_zk%2Fcc",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23316/i_zk/cc"}
   [junit4]   2> 124212 T372 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 124212 T384 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 126006 T346 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001/cores/collection1
   [junit4]   2> 126011 T346 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001
   [junit4]   2> 126012 T346 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 126014 T346 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:23334
   [junit4]   2> 126015 T346 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001/cores, hostPort=23333, solrconfig=solrconfig.xml, hostContext=/i_zk/cc, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/tempDir-001/jetty1}
   [junit4]   2> 126016 T346 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 126016 T346 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001/'
   [junit4]   2> 126082 T346 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001/solr.xml
   [junit4]   2> 126165 T346 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001/cores
   [junit4]   2> 126166 T346 oasc.CoreContainer.<init> New CoreContainer 294585126
   [junit4]   2> 126167 T346 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001/]
   [junit4]   2> 126167 T346 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001/lib
   [junit4]   2> 126168 T346 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001/lib).
   [junit4]   2> 126182 T346 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 126183 T346 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 126183 T346 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 126184 T346 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 126184 T346 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 126184 T346 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 126185 T346 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 126185 T346 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 126186 T346 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 126186 T346 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 126186 T346 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 126187 T346 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 126187 T346 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 126188 T346 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 126188 T346 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 126189 T346 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:23310/solr
   [junit4]   2> 126189 T346 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 126189 T346 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 126190 T346 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 126192 T399 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@78e4cf1a name:ZooKeeperConnection Watcher:127.0.0.1:23310 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 126193 T346 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 126194 T346 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 126196 T346 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 126198 T402 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@22a30e91 name:ZooKeeperConnection Watcher:127.0.0.1:23310/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 126198 T346 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 126204 T346 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 127212 T346 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:23333_i_zk%2Fcc
   [junit4]   2> 127213 T346 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:23333_i_zk%2Fcc
   [junit4]   2> 127216 T346 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 127220 T346 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001/cores
   [junit4]   2> 127222 T346 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001/cores/collection1/, dataDir=data/, transient=false, config=solrconfig.xml, shard=, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001/cores/collection1, coreNodeName=, name=collection1, loadOnStartup=true}
   [junit4]   2> 127223 T346 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001/cores/collection1/
   [junit4]   2> 127223 T346 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 127226 T403 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 127226 T403 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 127228 T372 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 127229 T403 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 127229 T403 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 127229 T373 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:23333_i_zk%2Fcc",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23333/i_zk/cc"} current state version: 4
   [junit4]   2> 127230 T403 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 127230 T373 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:23333_i_zk%2Fcc",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23333/i_zk/cc"}
   [junit4]   2> 127230 T373 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 127231 T373 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 127233 T373 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 127237 T403 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 127237 T403 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 127238 T403 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 127238 T403 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 127238 T403 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 127238 T403 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 127239 T403 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 127239 T403 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001/cores/collection1/'
   [junit4]   2> 127252 T403 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 127254 T403 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 127263 T403 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 127280 T403 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 127318 T403 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 127319 T403 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 127329 T403 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 127637 T403 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 127650 T403 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 127652 T403 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 127665 T403 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 127669 T403 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 127672 T403 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 127673 T403 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 127673 T403 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 127673 T403 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 127674 T403 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 127674 T403 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 127675 T403 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 127675 T403 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 127675 T403 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 127675 T403 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3f430afc
   [junit4]   2> 127676 T403 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001/cores/collection1/data
   [junit4]   2> 127676 T403 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 127677 T403 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 127677 T403 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 46171878E3F1D9B8-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 127677 T403 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=32, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.45963115368360286]
   [junit4]   2> 127678 T403 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1dd9940f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@37877418),segFN=segments_1,generation=1}
   [junit4]   2> 127678 T403 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 127685 T403 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 127685 T403 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 127686 T403 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 127686 T403 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 127686 T403 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 127686 T403 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 127687 T403 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 127687 T403 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 127687 T403 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 127701 T403 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 127703 T403 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 127704 T403 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 127706 T403 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 127713 T403 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/plugins,/admin/segments,/admin/mbeans,/admin/ping,/update/csv,/update/json,/get,/config,/admin/threads,standard,/replication,/admin/logging,/admin/properties,/update,/update/json/docs,/admin/file,/admin/luke,/admin/system,/schema
   [junit4]   2> 127714 T403 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 127715 T403 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 127716 T403 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 127717 T403 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 127717 T403 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 127718 T403 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=15, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6488594503462402]
   [junit4]   2> 127719 T403 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1dd9940f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@37877418),segFN=segments_1,generation=1}
   [junit4]   2> 127719 T403 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 127719 T403 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@746186b1[collection1] main
   [junit4]   2> 127719 T403 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 127721 T403 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 127721 T403 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 127721 T403 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 127721 T403 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 127722 T403 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 127722 T403 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 127722 T403 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 127723 T403 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 127723 T403 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 127724 T404 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@746186b1[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 127725 T403 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 127725 T407 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:23333/i_zk/cc collection:collection1 shard:shard1
   [junit4]   2> 127726 T346 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 127726 T346 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 127726 T407 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 127732 T407 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 127734 T372 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 127734 T407 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 127734 T407 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C39 name=collection1 org.apache.solr.core.SolrCore@cde2a73 url=http://127.0.0.1:23333/i_zk/cc/collection1 node=127.0.0.1:23333_i_zk%2Fcc C39_STATE=coll:collection1 core:collection1 props:{node_name=127.0.0.1:23333_i_zk%2Fcc, state=down, core=collection1, base_url=http://127.0.0.1:23333/i_zk/cc}
   [junit4]   2> 127735 T407 C:collection1 S:shard1 C39 P23333 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:23333/i_zk/cc/collection1/
   [junit4]   2> 127735 T407 C:collection1 S:shard1 C39 P23333 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 127735 T373 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 4
   [junit4]   2> 127735 T407 C:collection1 S:shard1 C39 P23333 oasc.SyncStrategy.syncToMe http://127.0.0.1:23333/i_zk/cc/collection1/ has no replicas
   [junit4]   2> 127736 T407 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:23333/i_zk/cc/collection1/ shard1
   [junit4]   2> 127736 T407 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 127737 T373 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 127738 T402 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 127739 T402 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 127743 T373 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23333/i_zk/cc",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 4
   [junit4]   2> 127745 T373 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 127745 T402 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 127746 T402 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 127748 T372 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 127794 T407 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:23333/i_zk/cc/collection1/ and leader is http://127.0.0.1:23333/i_zk/cc/collection1/
   [junit4]   2> 127795 T407 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:23333/i_zk/cc
   [junit4]   2> 127795 T407 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 127795 T407 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 127795 T407 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 127797 T372 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 127798 T373 oasc.Overseer$ClusterStateUpdater.run 

[...truncated too long message...]

.ZkTestServer.send4LetterWord connecting to 127.0.0.1:16134 16134
   [junit4]   2> 1467795 T4297 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/aliases.json
   [junit4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 		4	/solr/collections/collection1/state.json
   [junit4]   2> 		3	/solr/collections/c8n_1x3_commits/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1467795 T4296 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16199/
   [junit4]   2> 1467796 T4296 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16160/
   [junit4]   2> 1467796 T4296 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 12 connections to: http://127.0.0.1:16188/
   [junit4]   2> 1467796 T4296 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16214/
   [junit4]   2> 1467797 T4296 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:16230/
   [junit4]   2> 1467797 T4296 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16199/
   [junit4]   2> 1467798 T4296 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16160/
   [junit4]   2> 1467798 T4296 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16188/
   [junit4]   2> 1467798 T4296 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16214/
   [junit4]   2> 1467799 T4296 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16230/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderInitiatedRecoveryOnCommitTest -Dtests.method=test -Dtests.seed=46171878E3F1D9B8 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=en_PH -Dtests.timezone=America/Iqaluit -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   77.1s J1 | LeaderInitiatedRecoveryOnCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:16230/c8n_1x3_commits_shard1_replica1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([46171878E3F1D9B8:CE4327A24D0DB440]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 48 more
   [junit4]   2> 1467819 T4296 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 46171878E3F1D9B8-001
   [junit4]   2> 77083 T4295 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=DefaultSimilarity, locale=en_PH, timezone=America/Iqaluit
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=139158576,total=333447168
   [junit4]   2> NOTE: All tests run in this JVM: [TestLeaderElectionZkExpiry, TestManagedSchema, TermsComponentTest, CopyFieldTest, MinimalSchemaTest, SolrIndexSplitterTest, CursorPagingTest, TestClassNameShortening, TestFastLRUCache, TestMiniSolrCloudClusterSSL, RequestLoggingTest, SpellCheckComponentTest, HighlighterMaxOffsetTest, BadIndexSchemaTest, CoreMergeIndexesAdminHandlerTest, RecoveryAfterSoftCommitTest, CSVRequestHandlerTest, FastVectorHighlighterTest, TestSortingResponseWriter, CustomCollectionTest, DistribCursorPagingTest, AddBlockUpdateTest, BasicFunctionalityTest, OverseerRolesTest, TestWordDelimiterFilterFactory, SpatialRPTFieldTypeTest, TestLazyCores, DistributedExpandComponentTest, TestReplicationHandler, SuggesterWFSTTest, UniqFieldsUpdateProcessorFactoryTest, TestRecovery, TestConfigSets, TestCoreDiscovery, RequestHandlersTest, LeaderFailoverAfterPartitionTest, PolyFieldTest, TestCollationField, TestManagedSchemaFieldResource, TestStressReorder, TestStressRecovery, TestBulkSchemaConcurrent, TestJettySolrRunner, TestComponentsName, TestSolrDynamicMBean, TestCharFilters, OverseerCollectionProcessorTest, TestDistributedGrouping, TestExceedMaxTermLength, SearchHandlerTest, SpellingQueryConverterTest, TestPartialUpdateDeduplication, TimeZoneUtilsTest, JsonLoaderTest, TestRealTimeGet, LukeRequestHandlerTest, TestSweetSpotSimilarityFactory, TestLMJelinekMercerSimilarityFactory, AliasIntegrationTest, FullSolrCloudDistribCmdsTest, LeaderInitiatedRecoveryOnCommitTest]
   [junit4] Completed on J1 in 78.13s, 1 test, 1 error <<< FAILURES!

[...truncated 695 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 52 minutes 36 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 213069433 bytes
Compression is 0.0%
Took 40 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2881 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2881/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:59379/e_/tc/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:59379/e_/tc/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([9B66993D745958DD:1332A6E7DAA53525]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:46914/ngb/h/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:46914/ngb/h/collection1
	at __randomizedtesting.SeedInfo.seed([9B66993D745958DD:1332A6E7DAA53525]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:46914/ngb/h/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19404/zt_/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19404/zt_/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([9B66993D745958DD:1332A6E7DAA53525]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19404/zt_/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9235 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/init-core-data-001
   [junit4]   2> 64852 T106 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /zt_/
   [junit4]   2> 64909 T106 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 64922 T107 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 65425 T106 oasc.ZkTestServer.run start zk server on port:21233
   [junit4]   2> 65641 T106 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 65994 T106 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 66226 T114 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1f39201f name:ZooKeeperConnection Watcher:127.0.0.1:21233 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 66227 T106 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 66227 T106 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 66242 T106 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 66405 T106 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 66406 T106 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 66407 T117 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@60874ad8 name:ZooKeeperConnection Watcher:127.0.0.1:21233/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 66408 T106 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 66408 T106 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 66419 T106 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 66423 T106 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 66426 T106 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 66428 T106 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 66432 T106 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 66433 T106 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 66438 T106 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 66438 T106 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 66442 T106 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 66442 T106 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 66445 T106 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 66446 T106 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 66449 T106 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 66449 T106 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 66453 T106 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 66453 T106 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 66457 T106 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 66457 T106 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 66460 T106 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 66461 T106 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 66464 T106 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 66465 T106 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 66468 T106 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 66468 T106 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 66471 T106 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 66472 T106 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 67729 T106 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/control-001/cores/collection1
   [junit4]   2> 67733 T106 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 67736 T106 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:21000
   [junit4]   2> 67736 T106 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/control-001/cores, hostPort=10802, hostContext=/zt_, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/tempDir-001/control/data}
   [junit4]   2> 67737 T106 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@5fb271f1
   [junit4]   2> 67737 T106 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/control-001/'
   [junit4]   2> 67769 T106 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/control-001/solr.xml
   [junit4]   2> 67832 T106 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/control-001/cores
   [junit4]   2> 67833 T106 oasc.CoreContainer.<init> New CoreContainer 14898748
   [junit4]   2> 67833 T106 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/control-001/]
   [junit4]   2> 67834 T106 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/control-001/lib
   [junit4]   2> 67834 T106 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/control-001/lib).
   [junit4]   2> 67846 T106 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 67846 T106 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 67846 T106 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 67847 T106 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 67847 T106 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 67847 T106 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 67847 T106 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 67847 T106 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 67848 T106 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 67848 T106 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 67848 T106 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 67849 T106 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 67849 T106 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 67849 T106 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 67850 T106 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 67850 T106 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:21233/solr
   [junit4]   2> 67884 T106 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 67885 T106 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 67885 T106 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 67889 T129 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1eaa58ff name:ZooKeeperConnection Watcher:127.0.0.1:21233 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 67890 T106 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 67891 T106 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 67905 T106 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 67907 T132 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@165545e8 name:ZooKeeperConnection Watcher:127.0.0.1:21233/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 67907 T106 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 67929 T106 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 67946 T106 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 67958 T106 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 67962 T106 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 67966 T106 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 67995 T106 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 67999 T106 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 68002 T106 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 68004 T106 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:10802_zt_
   [junit4]   2> 68013 T106 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:10802_zt_
   [junit4]   2> 68017 T106 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 68020 T106 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 68023 T106 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 68053 T106 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:10802_zt_
   [junit4]   2> 68054 T106 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 68057 T106 oasc.Overseer.start Overseer (id=93587679924649987-127.0.0.1:10802_zt_-n_0000000000) starting
   [junit4]   2> 68079 T106 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 68125 T106 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 68147 T134 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 68147 T106 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 68171 T133 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 68183 T106 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/control-001/cores
   [junit4]   2> 68198 T106 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/control-001/cores/collection1, collection=control_collection, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/control-001/cores/collection1/, config=solrconfig.xml, transient=false, name=collection1, dataDir=data/, coreNodeName=, shard=, loadOnStartup=true, schema=schema.xml}
   [junit4]   2> 68199 T106 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/control-001/cores/collection1/
   [junit4]   2> 68199 T106 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 68201 T136 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 68201 T136 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 68219 T132 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 68219 T136 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 68225 T133 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:10802_zt_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:10802/zt_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1"} current state version: 0
   [junit4]   2> 68263 T133 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:10802_zt_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:10802/zt_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 68269 T133 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 68309 T133 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 68324 T132 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 69221 T136 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 69221 T136 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 69223 T136 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 69223 T136 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 69225 T136 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 69226 T136 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/control-001/cores/collection1/'
   [junit4]   2> 69317 T136 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 69357 T136 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 69410 T136 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 69452 T136 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 69532 T136 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 69534 T136 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 69558 T136 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 70139 T136 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 70164 T136 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 70170 T136 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 70243 T136 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 70250 T136 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 70257 T136 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 70259 T136 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 70259 T136 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 70259 T136 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 70261 T136 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 70261 T136 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 70262 T136 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 70262 T136 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 70263 T136 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/control-001/cores/collection1/data/
   [junit4]   2> 70263 T136 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@15185382
   [junit4]   2> 70284 T136 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/control-001/cores/collection1/data
   [junit4]   2> 70284 T136 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/control-001/cores/collection1/data/index/
   [junit4]   2> 70284 T136 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 70285 T136 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/control-001/cores/collection1/data/index
   [junit4]   2> 70285 T136 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=35, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 70287 T136 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@57cea843 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4088f3be),segFN=segments_1,generation=1}
   [junit4]   2> 70287 T136 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 70301 T136 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 70302 T136 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 70302 T136 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 70302 T136 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 70303 T136 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 70303 T136 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 70304 T136 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 70304 T136 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 70304 T136 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 70329 T136 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 70341 T136 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 70345 T136 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 70349 T136 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 70365 T136 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/segments,/admin/ping,/admin/mbeans,/admin/logging,/get,/admin/system,/admin/luke,/admin/properties,/admin/plugins,standard,/schema,/update,/update/json,/update/json/docs,/admin/file,/replication,/update/csv,/config,/admin/threads
   [junit4]   2> 70368 T136 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 70370 T136 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 70370 T136 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 70371 T136 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 70372 T136 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 70373 T136 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=31, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8930320413430614]
   [junit4]   2> 70374 T136 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@57cea843 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4088f3be),segFN=segments_1,generation=1}
   [junit4]   2> 70374 T136 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 70375 T136 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@5854278[collection1] main
   [junit4]   2> 70375 T136 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 70377 T136 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 70377 T136 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 70390 T136 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 70391 T136 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 70391 T136 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 70392 T136 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 70393 T136 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 70393 T136 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 70393 T136 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 70397 T137 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5854278[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 70406 T136 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 70407 T140 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:10802/zt_ collection:control_collection shard:shard1
   [junit4]   2> 70408 T106 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 70409 T106 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 70448 T140 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 70459 T140 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 70463 T132 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 70464 T140 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 70464 T140 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 70464 T133 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C16 name=collection1 org.apache.solr.core.SolrCore@3cbbaaf4 url=http://127.0.0.1:10802/zt_/collection1 node=127.0.0.1:10802_zt_ C16_STATE=coll:control_collection core:collection1 props:{state=down, node_name=127.0.0.1:10802_zt_, base_url=http://127.0.0.1:10802/zt_, core=collection1}
   [junit4]   2> 70473 T140 C:control_collection S:shard1 C16 P10802 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:10802/zt_/collection1/
   [junit4]   2> 70474 T140 C:control_collection S:shard1 C16 P10802 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 70474 T140 C:control_collection S:shard1 C16 P10802 oasc.SyncStrategy.syncToMe http://127.0.0.1:10802/zt_/collection1/ has no replicas
   [junit4]   2> 70475 T140 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:10802/zt_/collection1/ shard1
   [junit4]   2> 70476 T106 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 70477 T106 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 70479 T132 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 70480 T144 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5c14f4c name:ZooKeeperConnection Watcher:127.0.0.1:21233/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 70480 T106 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 70482 T106 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 70482 T106 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 70490 T140 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 70491 T106 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 70497 T132 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 70499 T133 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:10802/zt_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 70606 T144 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 70606 T132 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 70652 T140 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:10802/zt_/collection1/ and leader is http://127.0.0.1:10802/zt_/collection1/
   [junit4]   2> 70652 T140 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:10802/zt_
   [junit4]   2> 70652 T140 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 70653 T140 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 70653 T140 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 70656 T132 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 70658 T133 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:10802_zt_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:10802/zt_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1"} current state version: 3
   [junit4]   2> 70659 T133 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:10802_zt_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:10802/zt_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 70766 T132 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 70766 T144 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 71975 T106 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001/cores/collection1
   [junit4]   2> 71978 T106 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001
   [junit4]   2> 71978 T106 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 71981 T106 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:35143
   [junit4]   2> 71981 T106 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001/cores, solrconfig=solrconfig.xml, hostPort=13397, hostContext=/zt_, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/tempDir-001/jetty1}
   [junit4]   2> 71982 T106 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@5fb271f1
   [junit4]   2> 71982 T106 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001/'
   [junit4]   2> 72013 T106 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001/solr.xml
   [junit4]   2> 72053 T106 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001/cores
   [junit4]   2> 72053 T106 oasc.CoreContainer.<init> New CoreContainer 2009355367
   [junit4]   2> 72053 T106 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001/]
   [junit4]   2> 72054 T106 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001/lib
   [junit4]   2> 72054 T106 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001/lib).
   [junit4]   2> 72064 T106 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 72064 T106 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 72065 T106 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 72065 T106 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 72065 T106 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 72065 T106 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 72065 T106 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 72066 T106 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 72066 T106 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 72066 T106 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 72066 T106 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 72067 T106 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 72067 T106 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 72068 T106 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 72068 T106 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 72068 T106 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:21233/solr
   [junit4]   2> 72068 T106 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 72069 T106 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 72069 T106 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 72071 T156 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@35896b6 name:ZooKeeperConnection Watcher:127.0.0.1:21233 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 72071 T106 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 72072 T106 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 72074 T106 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 72076 T159 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7e308c04 name:ZooKeeperConnection Watcher:127.0.0.1:21233/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 72076 T106 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 72082 T106 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 73090 T106 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:13397_zt_
   [junit4]   2> 73092 T106 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:13397_zt_
   [junit4]   2> 73097 T106 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 73107 T106 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001/cores
   [junit4]   2> 73110 T106 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {shard=, collection=collection1, loadOnStartup=true, schema=schema.xml, dataDir=data/, name=collection1, coreNodeName=, transient=false, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001/cores/collection1/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001/cores/collection1, config=solrconfig.xml}
   [junit4]   2> 73110 T106 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001/cores/collection1/
   [junit4]   2> 73111 T106 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 73112 T160 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 73112 T160 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 73114 T132 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 73114 T160 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 73115 T133 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:13397_zt_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:13397/zt_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1"} current state version: 4
   [junit4]   2> 73116 T133 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:13397_zt_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:13397/zt_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 73117 T133 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 73117 T133 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard3
   [junit4]   2> 73120 T159 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 73120 T132 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 73120 T144 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 74114 T160 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 74115 T160 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 74116 T160 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 74116 T160 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 74118 T160 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 74118 T160 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001/cores/collection1/'
   [junit4]   2> 74138 T160 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 74141 T160 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 74156 T160 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 74184 T160 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 74264 T160 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 74267 T160 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 74289 T160 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 74788 T160 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 74810 T160 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 74814 T160 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 74845 T160 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 74852 T160 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 74858 T160 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 74860 T160 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 74860 T160 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 74860 T160 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 74862 T160 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 74863 T160 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 74863 T160 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 74863 T160 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 74864 T160 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 74864 T160 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@15185382
   [junit4]   2> 74865 T160 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001/cores/collection1/data
   [junit4]   2> 74866 T160 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 74866 T160 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 74867 T160 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 74867 T160 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=35, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 74868 T160 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@49f45c33 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@49d13e33),segFN=segments_1,generation=1}
   [junit4]   2> 74869 T160 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 74884 T160 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 74885 T160 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 74885 T160 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 74886 T160 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 74886 T160 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 74886 T160 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 74887 T160 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 74887 T160 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 74887 T160 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 74910 T160 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 74914 T160 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 74917 T160 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 74921 T160 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 74960 T160 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/segments,/admin/ping,/admin/mbeans,/admin/logging,/get,/admin/system,/admin/luke,/admin/properties,/admin/plugins,standard,/schema,/update,/update/json,/update/json/docs,/admin/file,/replication,/update/csv,/config,/admin/threads
   [junit4]   2> 74963 T160 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 74964 T160 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 74965 T160 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 74966 T160 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 74967 T160 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 74969 T160 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=31, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8930320413430614]
   [junit4]   2> 74971 T160 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@49f45c33 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@49d13e33),segFN=segments_1,generation=1}
   [junit4]   2> 74971 T160 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 74971 T160 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@1c414bd8[collection1] main
   [junit4]   2> 74972 T160 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 74974 T160 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 74974 T160 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 74975 T160 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 74975 T160 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 74975 T160 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 74976 T160 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 74977 T160 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 74977 T160 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 74977 T160 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 74980 T161 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1c414bd8[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 74981 T160 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 74982 T164 C:collection1 S:shard3 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:13397/zt_ collection:collection1 shard:shard3
   [junit4]   2> 74983 T106 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 74984 T106 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 74984 T164 C:collection1 S:shard3 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard3/election
   [junit4]   2> 74994 T164 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard3
   [junit4]   2> 74997 T132 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 74998 T164 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 74998 T164 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C17 name=collection1 org.apache.solr.core.SolrCore@7f17f2e0 url=http://127.0.0.1:13397/zt_/collection1 node=127.0.0.1:13397_zt_ C17_STATE=coll:collection1 core:collection1 props:{state=down, node_name=127.0.0.1:13397_zt_, base_url=http://127.0.0.1:13397/zt_, core=collection1}
   [junit4]   2> 74998 T164 C:collection1 S:shard3 C17 P13397 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:13397/zt_/collection1/
   [junit4]   2> 74999 T164 C:collection1 S:shard3 C17 P13397 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 74999 T164 C:collection1 S:shard3 C17 P13397 oasc.SyncStrategy.syncToMe http://127.0.0.1:13397/zt_/collection1/ has no replicas
   [junit4]   2> 74999 T133 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 75000 T164 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:13397/zt_/collection1/ shard3
   [junit4]   2> 75000 T164 C:collection1 S:shard3 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard3
   [junit4]   2> 75003 T144 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 75003 T132 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 75003 T159 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 75011 T133 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:13397/zt_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 75015 T132 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 75017 T159 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 75017 T132 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 75017 T144 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 75062 T164 C:collection1 S:shard3 oasc.ZkController.register We are http://127.0.0.1:13397/zt_/collection1/ and leader is http://127.0.0.1:13397/zt_/collection1/
   [junit4]   2> 75063 T164 C:collection1 S:shard3 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:13397/zt_
   [junit4]   2> 75063 T164 C:collection1 S:shard3 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 75063 T164 C:collection1 S:shard3 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 75063 T164 C:collection1 S:shard3 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 75066 T132 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 75068 T133 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:13397_zt_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:13397/zt_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1"} current state version: 7
   [junit4]   2> 75069 T133 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:13397_zt_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:13397/zt_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 75177 T144 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 75177 T132 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 75177 T159 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 76699 T106 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-2-001/cores/collection1
   [junit4]   2> 76703 T106 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 9B66993D745958DD-001/shard-2-001
   [junit4]   2> 76704 T106 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 76707 T106 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:38060


[...truncated too long message...]

til.java:31)
   [junit4]   2> 		at org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:142)
   [junit4]   2> 		... 11 more
   [junit4]   2> 	
   [junit4]   2> 852947 T2693 C:control_collection S:shard1 c:collection1 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/e_/tc,null}
   [junit4]   2> 853041 T2693 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:59358 59358
   [junit4]   2> 853381 T2694 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:59358 59358
   [junit4]   2> 853382 T2694 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		6	/solr/aliases.json
   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 		3	/solr/collections/c8n_1x3_commits/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 		4	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 853383 T2693 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:59365/e_/tc
   [junit4]   2> 853383 T2693 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:59426/e_/tc
   [junit4]   2> 853383 T2693 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:59391/e_/tc
   [junit4]   2> 853383 T2693 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 14 connections to: http://127.0.0.1:59371/e_/tc
   [junit4]   2> 853384 T2693 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:59379/e_/tc
   [junit4]   2> 853384 T2693 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:59365/e_/tc
   [junit4]   2> 853384 T2693 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:59426/e_/tc
   [junit4]   2> 853384 T2693 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:59391/e_/tc
   [junit4]   2> 853384 T2693 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:59371/e_/tc
   [junit4]   2> 853385 T2693 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:59379/e_/tc
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderInitiatedRecoveryOnCommitTest -Dtests.method=test -Dtests.seed=9B66993D745958DD -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=cs_CZ -Dtests.timezone=Europe/Malta -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   83.5s J1 | LeaderInitiatedRecoveryOnCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:59379/e_/tc/c8n_1x3_commits_shard1_replica1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([9B66993D745958DD:1332A6E7DAA53525]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 48 more
   [junit4]   2> 853391 T2693 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 9B66993D745958DD-001
   [junit4]   2> 83493 T2692 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=DefaultSimilarity, locale=cs_CZ, timezone=Europe/Malta
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=103965640,total=296747008
   [junit4]   2> NOTE: All tests run in this JVM: [SpatialRPTFieldTypeTest, MigrateRouteKeyTest, TestReRankQParserPlugin, CustomCollectionTest, JsonLoaderTest, TestPostingsSolrHighlighter, RequestLoggingTest, TestClassNameShortening, TestFieldSortValues, DirectUpdateHandlerTest, OpenExchangeRatesOrgProviderTest, TestManagedSchemaFieldTypeResource, TermVectorComponentTest, RecoveryAfterSoftCommitTest, TestSolrQueryParserResource, MoreLikeThisHandlerTest, AnalysisErrorHandlingTest, ZkStateWriterTest, TestSchemaResource, TestManagedStopFilterFactory, InfoHandlerTest, TestMiniSolrCloudClusterSSL, BadComponentTest, TestStandardQParsers, HdfsRecoveryZkTest, CollectionsAPIDistributedZkTest, ChaosMonkeyNothingIsSafeTest, TestLFUCache, EchoParamsTest, SolrCoreTest, DistribJoinFromCollectionTest, UUIDFieldTest, ExternalFileFieldSortTest, TestUniqueKeyFieldResource, TestDistribDocBasedVersion, TestRandomFaceting, TestManagedSchema, FileBasedSpellCheckerTest, IndexSchemaTest, LeaderInitiatedRecoveryOnCommitTest]
   [junit4] Completed on J1 in 83.77s, 1 test, 1 error <<< FAILURES!

[...truncated 867 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 56 minutes 56 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 206578940 bytes
Compression is 0.0%
Took 44 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2880 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2880/

4 tests failed.
REGRESSION:  org.apache.solr.cloud.RecoveryZkTest.test

Error Message:
shard1 is not consistent.  Got 501 from http://127.0.0.1:39753/mkrd/collection1lastClient and got 264 from http://127.0.0.1:39784/mkrd/collection1

Stack Trace:
java.lang.AssertionError: shard1 is not consistent.  Got 501 from http://127.0.0.1:39753/mkrd/collection1lastClient and got 264 from http://127.0.0.1:39784/mkrd/collection1
	at __randomizedtesting.SeedInfo.seed([5E8BB946415F99B4:D6DF869CEFA3F44C]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.RecoveryZkTest.test(RecoveryZkTest.java:123)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:39306/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:39306/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([5E8BB946415F99B4:D6DF869CEFA3F44C]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:31114/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:31114/collection1
	at __randomizedtesting.SeedInfo.seed([5E8BB946415F99B4:D6DF869CEFA3F44C]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:31114/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:56201/repfacttest_c8n_1x3_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:56201/repfacttest_c8n_1x3_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([5E8BB946415F99B4:D6DF869CEFA3F44C]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:56201/repfacttest_c8n_1x3_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9464 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/init-core-data-001
   [junit4]   2> 269650 T884 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 269655 T884 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 269656 T885 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 269756 T884 oasc.ZkTestServer.run start zk server on port:31099
   [junit4]   2> 269757 T884 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 269758 T884 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 269761 T892 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@657ecaa0 name:ZooKeeperConnection Watcher:127.0.0.1:31099 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 269761 T884 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 269761 T884 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 269762 T884 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 269764 T884 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 269765 T884 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 269766 T895 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5b0f66d6 name:ZooKeeperConnection Watcher:127.0.0.1:31099/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 269766 T884 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 269766 T884 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 269766 T884 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 269768 T884 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 269769 T884 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 269770 T884 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 269772 T884 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 269772 T884 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 269775 T884 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 269775 T884 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 269777 T884 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 269777 T884 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 269779 T884 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 269779 T884 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 269781 T884 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 269781 T884 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 269783 T884 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 269783 T884 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 269785 T884 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 269785 T884 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 269787 T884 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 269787 T884 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 269789 T884 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 269789 T884 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 269791 T884 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 269792 T884 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 269793 T884 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 269794 T884 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 271412 T884 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/control-001/cores/collection1
   [junit4]   2> 271416 T884 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 271420 T884 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:31103
   [junit4]   2> 271420 T884 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/tempDir-001/control/data, hostPort=31102, hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/control-001/cores}
   [junit4]   2> 271421 T884 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3972aa3f
   [junit4]   2> 271421 T884 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/control-001/'
   [junit4]   2> 271469 T884 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/control-001/solr.xml
   [junit4]   2> 271529 T884 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/control-001/cores
   [junit4]   2> 271530 T884 oasc.CoreContainer.<init> New CoreContainer 914690658
   [junit4]   2> 271530 T884 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/control-001/]
   [junit4]   2> 271530 T884 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/control-001/lib
   [junit4]   2> 271531 T884 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/control-001/lib).
   [junit4]   2> 271541 T884 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 271541 T884 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 271542 T884 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 271542 T884 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 271542 T884 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 271542 T884 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 271542 T884 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 271543 T884 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 271543 T884 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 271543 T884 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 271543 T884 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 271544 T884 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 271544 T884 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 271544 T884 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 271544 T884 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 271545 T884 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:31099/solr
   [junit4]   2> 271545 T884 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 271545 T884 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 271546 T884 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 271547 T907 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@49765877 name:ZooKeeperConnection Watcher:127.0.0.1:31099 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 271548 T884 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 271548 T884 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 271550 T884 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 271551 T910 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@28fa04f3 name:ZooKeeperConnection Watcher:127.0.0.1:31099/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 271551 T884 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 271553 T884 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 271555 T884 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 271556 T884 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 271558 T884 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 271560 T884 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 271562 T884 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 271563 T884 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 271564 T884 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 271565 T884 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:31102_
   [junit4]   2> 271566 T884 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:31102_
   [junit4]   2> 271567 T884 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 271568 T884 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 271570 T884 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 271571 T884 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:31102_
   [junit4]   2> 271571 T884 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 271572 T884 oasc.Overseer.start Overseer (id=93586741821046787-127.0.0.1:31102_-n_0000000000) starting
   [junit4]   2> 271574 T884 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 271579 T884 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 271580 T912 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 271580 T884 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 271580 T911 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 271583 T884 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/control-001/cores
   [junit4]   2> 271585 T884 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {transient=false, config=solrconfig.xml, dataDir=data/, name=collection1, coreNodeName=, loadOnStartup=true, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/control-001/cores/collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/control-001/cores/collection1/, collection=control_collection, shard=, schema=schema.xml}
   [junit4]   2> 271586 T884 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/control-001/cores/collection1/
   [junit4]   2> 271586 T884 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 271587 T914 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 271587 T914 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 271588 T910 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 271588 T914 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 271589 T911 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:31102",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:31102_",
   [junit4]   2> 	  "state":"down"} current state version: 0
   [junit4]   2> 271589 T911 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:31102",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:31102_",
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 271589 T911 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 271589 T911 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 271590 T910 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 272589 T914 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 272590 T914 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 272590 T914 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 272590 T914 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 272591 T914 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 272591 T914 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/control-001/cores/collection1/'
   [junit4]   2> 272605 T914 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 272607 T914 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 272618 T914 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 272638 T914 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 272819 T914 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 272820 T914 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 272829 T914 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 273176 T914 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 273192 T914 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 273194 T914 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 273203 T914 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 273207 T914 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 273210 T914 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 273211 T914 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 273211 T914 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 273211 T914 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 273212 T914 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 273212 T914 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 273212 T914 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 273212 T914 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 273212 T914 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/control-001/cores/collection1/data/
   [junit4]   2> 273213 T914 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a27d7d0
   [junit4]   2> 273213 T914 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/control-001/cores/collection1/data
   [junit4]   2> 273213 T914 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/control-001/cores/collection1/data/index/
   [junit4]   2> 273214 T914 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 273214 T914 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/control-001/cores/collection1/data/index
   [junit4]   2> 273214 T914 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=39, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8675328596970137]
   [junit4]   2> 273215 T914 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@37bf1383 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@619a1a0c),segFN=segments_1,generation=1}
   [junit4]   2> 273215 T914 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 273219 T914 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 273219 T914 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 273219 T914 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 273219 T914 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 273219 T914 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 273220 T914 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 273220 T914 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 273220 T914 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 273220 T914 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 273247 T914 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 273249 T914 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 273251 T914 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 273253 T914 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 273256 T914 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update/csv,/admin/system,/update/json/docs,/admin/plugins,/admin/luke,/replication,standard,/get,/admin/mbeans,/admin/logging,/admin/segments,/update/json,/admin/threads,/config,/update,/admin/ping,/admin/file,/admin/properties,/schema
   [junit4]   2> 273258 T914 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 273260 T914 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 273260 T914 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 273262 T914 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 273262 T914 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 273262 T914 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=48, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7866241976657329]
   [junit4]   2> 273263 T914 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@37bf1383 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@619a1a0c),segFN=segments_1,generation=1}
   [junit4]   2> 273263 T914 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 273263 T914 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@35a6ab1d[collection1] main
   [junit4]   2> 273263 T914 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 273264 T914 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 273264 T914 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 273265 T914 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 273265 T914 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 273265 T914 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 273265 T914 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 273265 T914 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 273266 T914 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 273266 T914 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 273266 T915 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@35a6ab1d[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 273267 T914 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 273267 T918 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:31102 collection:control_collection shard:shard1
   [junit4]   2> 273267 T884 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 273268 T884 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 273268 T918 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 273271 T884 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 273271 T918 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 273272 T884 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 273273 T922 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@665b8cbf name:ZooKeeperConnection Watcher:127.0.0.1:31099/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 273273 T884 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 273273 T910 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 273273 T884 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 273273 T918 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 273273 T884 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 273274 T911 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 273274 T918 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C194 name=collection1 org.apache.solr.core.SolrCore@52c8671d url=http://127.0.0.1:31102/collection1 node=127.0.0.1:31102_ C194_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:31102, core=collection1, node_name=127.0.0.1:31102_, state=down}
   [junit4]   2> 273275 T918 C:control_collection S:shard1 C194 P31102 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:31102/collection1/
   [junit4]   2> 273275 T918 C:control_collection S:shard1 C194 P31102 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 273275 T918 C:control_collection S:shard1 C194 P31102 oasc.SyncStrategy.syncToMe http://127.0.0.1:31102/collection1/ has no replicas
   [junit4]   2> 273275 T918 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:31102/collection1/ shard1
   [junit4]   2> 273275 T918 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 273275 T910 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 273275 T922 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 273277 T884 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 273279 T910 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 273279 T911 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:31102",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 273382 T910 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 273382 T922 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 273432 T918 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:31102/collection1/ and leader is http://127.0.0.1:31102/collection1/
   [junit4]   2> 273433 T918 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:31102
   [junit4]   2> 273433 T918 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 273433 T918 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 273433 T918 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 273434 T910 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 273435 T911 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:31102",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:31102_",
   [junit4]   2> 	  "state":"active"} current state version: 3
   [junit4]   2> 273436 T911 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:31102",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:31102_",
   [junit4]   2> 	  "state":"active"}
   [junit4]   2> 273539 T910 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 273539 T922 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 274649 T884 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001/cores/collection1
   [junit4]   2> 274652 T884 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001
   [junit4]   2> 274653 T884 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 274656 T884 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:31115
   [junit4]   2> 274656 T884 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/tempDir-001/jetty1, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001/cores, hostContext=/, hostPort=31114, solrconfig=solrconfig.xml}
   [junit4]   2> 274656 T884 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3972aa3f
   [junit4]   2> 274657 T884 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001/'
   [junit4]   2> 274711 T884 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001/solr.xml
   [junit4]   2> 274784 T884 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001/cores
   [junit4]   2> 274784 T884 oasc.CoreContainer.<init> New CoreContainer 540568750
   [junit4]   2> 274785 T884 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001/]
   [junit4]   2> 274785 T884 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001/lib
   [junit4]   2> 274786 T884 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001/lib).
   [junit4]   2> 274798 T884 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 274799 T884 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 274799 T884 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 274799 T884 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 274799 T884 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 274800 T884 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 274800 T884 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 274800 T884 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 274800 T884 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 274801 T884 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 274801 T884 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 274802 T884 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 274802 T884 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 274802 T884 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 274802 T884 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 274803 T884 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:31099/solr
   [junit4]   2> 274803 T884 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 274803 T884 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 274804 T884 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 274806 T934 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4e178d9c name:ZooKeeperConnection Watcher:127.0.0.1:31099 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 274806 T884 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 274807 T884 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 274809 T884 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 274810 T937 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@335b4293 name:ZooKeeperConnection Watcher:127.0.0.1:31099/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 274810 T884 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 274815 T884 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 275820 T884 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:31114_
   [junit4]   2> 275821 T884 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:31114_
   [junit4]   2> 275823 T884 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 275825 T884 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001/cores
   [junit4]   2> 275827 T884 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {dataDir=data/, loadOnStartup=true, config=solrconfig.xml, name=collection1, schema=schema.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001/cores/collection1/, transient=false, collection=collection1, shard=, coreNodeName=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001/cores/collection1}
   [junit4]   2> 275828 T884 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001/cores/collection1/
   [junit4]   2> 275828 T884 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 275829 T938 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 275829 T938 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 275829 T938 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 275829 T910 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 275830 T911 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:31114",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:31114_",
   [junit4]   2> 	  "state":"down"} current state version: 4
   [junit4]   2> 275831 T911 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:31114",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:31114_",
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 275831 T911 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 275831 T911 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 275832 T922 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 275832 T937 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 275832 T910 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 276831 T938 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 276832 T938 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 276832 T938 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 276832 T938 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 276833 T938 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 276833 T938 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001/cores/collection1/'
   [junit4]   2> 276846 T938 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 276848 T938 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 276858 T938 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 276877 T938 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 276917 T938 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 276918 T938 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 276927 T938 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 277266 T938 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 277285 T938 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 277289 T938 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 277301 T938 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 277305 T938 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 277310 T938 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 277310 T938 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 277311 T938 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 277311 T938 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 277312 T938 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 277312 T938 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 277312 T938 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 277313 T938 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 277313 T938 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 277313 T938 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a27d7d0
   [junit4]   2> 277314 T938 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001/cores/collection1/data
   [junit4]   2> 277314 T938 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 277315 T938 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 277315 T938 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 5E8BB946415F99B4-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 277315 T938 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=39, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8675328596970137]
   [junit4]   2> 277316 T938 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3f1bc613 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@386929c3),segFN=segments_1,generation=1}
   [junit4]   2> 277316 T938 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 277323 T938 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 277324 T938 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 277324 T938 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 277324 T938 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 277325 T938 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 277325 T938 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 277326 T938 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 277326 T938 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 277326 T938 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 277356 T938 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 277358 T938 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 277361 T938 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 277364 T938 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 277372 T938 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update/csv,/admin/system,/update/json/docs,/admin/plugins,/admin/luke,/replication,standard,/get,/admin/mbeans,/admin/logging,/admin/segments,/update/json,/admin/threads,/config,/update,/admin/ping,/admin/file,/admin/properties,/schema
   [junit4]   2> 277374 T938 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 277376 T938 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 277376 T938 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 277378 T938 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 277378 T938 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 277379 T938 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=48, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7866241976657329]
   [junit4]   2> 277380 T938 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3f1bc613 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@386929c3),segFN=segments_1,generation=1}
   [junit4]   2> 277380 T938 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 277381 T938 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@71971b44[collection1] main
   [junit4]   2> 277381 T938 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 277382 T938 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 277382 T938 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 277382 T938 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 277383 T938 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 277383 T938 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 277383 T938 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 277384 T938 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 277384 T938 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 277384 T938 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 277385 T939 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@71971b44[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 277386 T938 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 277386 T942 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:31114 collection:collection1 shard:shard1
   [junit4]   2> 277387 T884 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 277387 T884 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 277387 T942 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 277392 T942 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 277393 T910 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 277393 T942 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 277394 T942 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C195 name=collection1 org.apache.solr.core.SolrCore@761f2384 url=http://127.0.0.1:31114/collection1 node=127.0.0.1:31114_ C195_STATE=coll:collection1 core:collection1 props:{base_url=http://127.0.0.1:31114, core=collection1, node_name=127.0.0.1:31114_, state=down}
   [junit4]   2> 277394 T942 C:collection1 S:shard1 C195 P31114 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:31114/collection1/
   [junit4]   2> 277394 T911 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 277394 T942 C:collection1 S:shard1 C195 P31114 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 277394 T942 C:collection1 S:shard1 C195 P31114 oasc.SyncStrategy.syncToMe http://127.0.0.1:31114/collection1/ has no replicas
   [junit4]   2> 277395 T942 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:31114/collection1/ sh

[...truncated too long message...]

13 T3106 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:39267 39267
   [junit4]   2> 1478809 T3107 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:39267 39267
   [junit4]   2> 1478810 T3107 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		6	/solr/aliases.json
   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 		3	/solr/collections/c8n_1x3_commits/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/queue
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1478811 T3106 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:39278/
   [junit4]   2> 1478811 T3106 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:39360/
   [junit4]   2> 1478811 T3106 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 3 connections to: http://127.0.0.1:39318/
   [junit4]   2> 1478811 T3106 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 13 connections to: http://127.0.0.1:39306/
   [junit4]   2> 1478812 T3106 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:39326/
   [junit4]   2> 1478812 T3106 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:39278/
   [junit4]   2> 1478812 T3106 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:39360/
   [junit4]   2> 1478812 T3106 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:39318/
   [junit4]   2> 1478813 T3106 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:39306/
   [junit4]   2> 1478813 T3106 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:39326/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderInitiatedRecoveryOnCommitTest -Dtests.method=test -Dtests.seed=5E8BB946415F99B4 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=es_BO -Dtests.timezone=IET -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   83.2s J0 | LeaderInitiatedRecoveryOnCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:39306/c8n_1x3_commits_shard1_replica3
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([5E8BB946415F99B4:D6DF869CEFA3F44C]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 48 more
   [junit4]   2> 1478826 T3106 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 5E8BB946415F99B4-001
   [junit4]   2> 83169 T3105 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=DefaultSimilarity, locale=es_BO, timezone=IET
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=155916320,total=346030080
   [junit4]   2> NOTE: All tests run in this JVM: [JsonLoaderTest, StressHdfsTest, DistributedFacetPivotLargeTest, DateMathParserTest, CurrencyFieldOpenExchangeTest, DistribJoinFromCollectionTest, FacetPivotSmallTest, TestLRUCache, TestHighlightDedupGrouping, OverseerCollectionProcessorTest, AtomicUpdatesTest, TestSweetSpotSimilarityFactory, TestSimpleTrackingShardHandler, BasicZkTest, SolrIndexSplitterTest, TestCSVResponseWriter, TestSchemaManager, SolrTestCaseJ4Test, TestRandomFaceting, TestBadConfig, CloudMLTQParserTest, TestStressVersions, NumericFieldsTest, TestReplicationHandlerBackup, QueryResultKeyTest, DateFieldTest, TestRTGBase, SpellingQueryConverterTest, TestWordDelimiterFilterFactory, ParsingFieldUpdateProcessorsTest, TestRecoveryHdfs, RecoveryAfterSoftCommitTest, CoreAdminCreateDiscoverTest, SearchHandlerTest, TestUpdate, TestAnalyzedSuggestions, TestPhraseSuggestions, TestRecovery, FastVectorHighlighterTest, SOLR749Test, TestFieldTypeResource, DocumentAnalysisRequestHandlerTest, TestPHPSerializedResponseWriter, ResponseHeaderTest, TestHashQParserPlugin, TestInitQParser, TestJettySolrRunner, SimpleFacetsTest, TestStressUserVersions, BadComponentTest, TestManagedResourceStorage, FullSolrCloudDistribCmdsTest, TestSchemaResource, MultiThreadedOCPTest, TestStressReorder, TestOmitPositions, TestExceedMaxTermLength, HttpPartitionTest, ShardSplitTest, LeaderInitiatedRecoveryOnCommitTest]
   [junit4] Completed on J0 in 83.71s, 1 test, 1 error <<< FAILURES!

[...truncated 722 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 1 failure, 61 ignored (21 assumptions)

Total time: 53 minutes 59 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 215712534 bytes
Compression is 0.0%
Took 40 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2879 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2879/

4 tests failed.
FAILED:  org.apache.solr.handler.component.DistributedMLTComponentTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:33912//collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:33912//collection1
	at __randomizedtesting.SeedInfo.seed([94BF5CA3001FF936:1CEB6379AEE394CE]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:566)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:943)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:558)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:606)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:588)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:567)
	at org.apache.solr.handler.component.DistributedMLTComponentTest.test(DistributedMLTComponentTest.java:126)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 51 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:57562/f/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:57562/f/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([94BF5CA3001FF936:1CEB6379AEE394CE]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:51163/p_myp/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:51163/p_myp/collection1
	at __randomizedtesting.SeedInfo.seed([94BF5CA3001FF936:1CEB6379AEE394CE]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:51163/p_myp/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:57725/z/su/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:57725/z/su/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([94BF5CA3001FF936:1CEB6379AEE394CE]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:57725/z/su/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9675 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/init-core-data-001
   [junit4]   2> 444750 T798 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /p_myp/
   [junit4]   2> 444754 T798 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 444755 T799 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 444855 T798 oasc.ZkTestServer.run start zk server on port:51149
   [junit4]   2> 444856 T798 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 444857 T798 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 444861 T806 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@71cc5eb5 name:ZooKeeperConnection Watcher:127.0.0.1:51149 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 444861 T798 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 444862 T798 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 444862 T798 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 444864 T798 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 444865 T798 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 444867 T809 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5611f639 name:ZooKeeperConnection Watcher:127.0.0.1:51149/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 444867 T798 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 444867 T798 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 444868 T798 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 444870 T798 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 444872 T798 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 444873 T798 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 444875 T798 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 444875 T798 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 444878 T798 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 444879 T798 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 444981 T798 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 444981 T798 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 444983 T798 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 444983 T798 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 444985 T798 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 444986 T798 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 444987 T798 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 444988 T798 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 444989 T798 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 444990 T798 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 444991 T798 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 444992 T798 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 444994 T798 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 444994 T798 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 444996 T798 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 444996 T798 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 444998 T798 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 444999 T798 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 446542 T798 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/control-001/cores/collection1
   [junit4]   2> 446546 T798 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 446549 T798 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:51154
   [junit4]   2> 446550 T798 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/p_myp, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/control-001/cores, hostPort=51153, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/tempDir-001/control/data}
   [junit4]   2> 446550 T798 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 446551 T798 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/control-001/'
   [junit4]   2> 446585 T798 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/control-001/solr.xml
   [junit4]   2> 446626 T798 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/control-001/cores
   [junit4]   2> 446626 T798 oasc.CoreContainer.<init> New CoreContainer 1742799638
   [junit4]   2> 446627 T798 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/control-001/]
   [junit4]   2> 446627 T798 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/control-001/lib
   [junit4]   2> 446627 T798 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/control-001/lib).
   [junit4]   2> 446637 T798 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 446637 T798 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 446637 T798 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 446638 T798 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 446638 T798 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 446638 T798 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 446638 T798 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 446638 T798 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 446639 T798 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 446639 T798 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 446639 T798 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 446640 T798 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 446640 T798 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 446640 T798 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 446640 T798 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 446640 T798 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51149/solr
   [junit4]   2> 446641 T798 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 446641 T798 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 446642 T798 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 446643 T821 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2d84ba1b name:ZooKeeperConnection Watcher:127.0.0.1:51149 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 446644 T798 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 446644 T798 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 446646 T798 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 446656 T824 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3b3019d2 name:ZooKeeperConnection Watcher:127.0.0.1:51149/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 446657 T798 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 446658 T798 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 446660 T798 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 446661 T798 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 446663 T798 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 446664 T798 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 446666 T798 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 446667 T798 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 446668 T798 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 446669 T798 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51153_p_myp
   [junit4]   2> 446670 T798 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:51153_p_myp
   [junit4]   2> 446671 T798 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 446672 T798 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 446673 T798 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 446674 T798 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:51153_p_myp
   [junit4]   2> 446675 T798 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 446676 T798 oasc.Overseer.start Overseer (id=93586175975817219-127.0.0.1:51153_p_myp-n_0000000000) starting
   [junit4]   2> 446677 T798 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 446682 T798 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 446683 T826 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 446683 T798 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 446683 T825 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 446686 T798 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/control-001/cores
   [junit4]   2> 446688 T798 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, collection=control_collection, dataDir=data/, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/control-001/cores/collection1, shard=, coreNodeName=, name=collection1, loadOnStartup=true, transient=false, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/control-001/cores/collection1/}
   [junit4]   2> 446689 T798 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/control-001/cores/collection1/
   [junit4]   2> 446689 T798 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 446689 T828 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 446690 T828 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 446690 T828 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 446690 T824 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 446691 T825 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:51153_p_myp",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51153/p_myp",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null} current state version: 0
   [junit4]   2> 446691 T825 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:51153_p_myp",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51153/p_myp",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 446692 T825 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 446692 T825 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 446693 T824 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 447691 T828 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 447692 T828 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 447692 T828 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 447693 T828 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 447693 T828 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 447693 T828 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/control-001/cores/collection1/'
   [junit4]   2> 447705 T828 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 447707 T828 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 447715 T828 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 447729 T828 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 447762 T828 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 447863 T828 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 447871 T828 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 448179 T828 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 448195 T828 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 448198 T828 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 448210 T828 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 448215 T828 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 448219 T828 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 448220 T828 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 448220 T828 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 448220 T828 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 448221 T828 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 448221 T828 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 448222 T828 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 448222 T828 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 448222 T828 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/control-001/cores/collection1/data/
   [junit4]   2> 448223 T828 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@78a8ee09
   [junit4]   2> 448224 T828 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/control-001/cores/collection1/data
   [junit4]   2> 448224 T828 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/control-001/cores/collection1/data/index/
   [junit4]   2> 448224 T828 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 448225 T828 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/control-001/cores/collection1/data/index
   [junit4]   2> 448225 T828 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=18, maxMergeAtOnceExplicit=29, maxMergedSegmentMB=19.7001953125, floorSegmentMB=1.7607421875, forceMergeDeletesPctAllowed=20.3593349700404, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8547633598335697
   [junit4]   2> 448226 T828 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@77eb2a1b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7019901e),segFN=segments_1,generation=1}
   [junit4]   2> 448226 T828 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 448233 T828 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 448233 T828 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 448233 T828 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 448234 T828 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 448234 T828 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 448234 T828 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 448235 T828 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 448235 T828 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 448235 T828 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 448255 T828 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 448257 T828 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 448260 T828 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 448262 T828 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 448268 T828 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update/json/docs,/get,/admin/properties,/config,/admin/logging,/admin/threads,standard,/admin/system,/admin/ping,/admin/segments,/admin/plugins,/update/csv,/admin/file,/admin/mbeans,/schema,/update,/admin/luke,/replication,/update/json
   [junit4]   2> 448270 T828 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 448272 T828 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 448272 T828 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 448273 T828 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 448274 T828 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 448275 T828 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1118247169, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 448276 T828 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@77eb2a1b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7019901e),segFN=segments_1,generation=1}
   [junit4]   2> 448276 T828 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 448276 T828 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@7a9e0d6d[collection1] main
   [junit4]   2> 448277 T828 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 448278 T828 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 448278 T828 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 448279 T828 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 448279 T828 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 448279 T828 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 448280 T828 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 448280 T828 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 448280 T828 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 448280 T828 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 448281 T829 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7a9e0d6d[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 448282 T828 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 448283 T832 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:51153/p_myp collection:control_collection shard:shard1
   [junit4]   2> 448283 T798 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 448283 T798 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 448283 T832 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 448286 T798 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 448287 T798 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 448288 T832 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 448289 T836 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@15489fd2 name:ZooKeeperConnection Watcher:127.0.0.1:51149/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 448289 T798 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 448290 T824 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 448290 T798 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 448290 T832 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 448291 T832 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 448290 T798 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> ASYNC  NEW_CORE C269 name=collection1 org.apache.solr.core.SolrCore@743f1f8a url=http://127.0.0.1:51153/p_myp/collection1 node=127.0.0.1:51153_p_myp C269_STATE=coll:control_collection core:collection1 props:{state=down, core=collection1, node_name=127.0.0.1:51153_p_myp, base_url=http://127.0.0.1:51153/p_myp}
   [junit4]   2> 448291 T832 C:control_collection S:shard1 C269 P51153 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:51153/p_myp/collection1/
   [junit4]   2> 448291 T825 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 448292 T832 C:control_collection S:shard1 C269 P51153 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 448292 T832 C:control_collection S:shard1 C269 P51153 oasc.SyncStrategy.syncToMe http://127.0.0.1:51153/p_myp/collection1/ has no replicas
   [junit4]   2> 448293 T832 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:51153/p_myp/collection1/ shard1
   [junit4]   2> 448293 T832 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 448294 T836 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 448294 T824 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 448296 T798 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 448297 T824 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 448298 T825 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51153/p_myp",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 448401 T836 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 448401 T824 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 448450 T832 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:51153/p_myp/collection1/ and leader is http://127.0.0.1:51153/p_myp/collection1/
   [junit4]   2> 448451 T832 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:51153/p_myp
   [junit4]   2> 448451 T832 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 448451 T832 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 448451 T832 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 448452 T824 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 448453 T825 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:51153_p_myp",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51153/p_myp",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null} current state version: 3
   [junit4]   2> 448454 T825 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:51153_p_myp",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51153/p_myp",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 448557 T824 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 448557 T836 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 449531 T798 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001/cores/collection1
   [junit4]   2> 449534 T798 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001
   [junit4]   2> 449535 T798 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 449537 T798 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:51164
   [junit4]   2> 449537 T798 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solrconfig=solrconfig.xml, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/tempDir-001/jetty1, hostPort=51163, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001/cores, hostContext=/p_myp}
   [junit4]   2> 449537 T798 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 449538 T798 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001/'
   [junit4]   2> 449567 T798 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001/solr.xml
   [junit4]   2> 449606 T798 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001/cores
   [junit4]   2> 449606 T798 oasc.CoreContainer.<init> New CoreContainer 610610784
   [junit4]   2> 449606 T798 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001/]
   [junit4]   2> 449606 T798 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001/lib
   [junit4]   2> 449607 T798 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001/lib).
   [junit4]   2> 449617 T798 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 449617 T798 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 449617 T798 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 449618 T798 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 449618 T798 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 449618 T798 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 449618 T798 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 449619 T798 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 449619 T798 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 449619 T798 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 449619 T798 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 449620 T798 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 449620 T798 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 449620 T798 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 449620 T798 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 449621 T798 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51149/solr
   [junit4]   2> 449621 T798 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 449621 T798 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 449622 T798 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 449623 T848 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@31e57021 name:ZooKeeperConnection Watcher:127.0.0.1:51149 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 449623 T798 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 449624 T798 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 449626 T798 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 449627 T851 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@72932842 name:ZooKeeperConnection Watcher:127.0.0.1:51149/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 449627 T798 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 449631 T798 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 450634 T798 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51163_p_myp
   [junit4]   2> 450635 T798 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:51163_p_myp
   [junit4]   2> 450637 T798 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 450640 T798 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001/cores
   [junit4]   2> 450642 T798 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001/cores/collection1/, name=collection1, config=solrconfig.xml, dataDir=data/, collection=collection1, transient=false, loadOnStartup=true, coreNodeName=, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001/cores/collection1, shard=}
   [junit4]   2> 450643 T798 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001/cores/collection1/
   [junit4]   2> 450643 T798 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 450645 T852 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 450645 T852 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 450646 T824 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 450646 T852 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 450647 T825 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:51163_p_myp",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51163/p_myp",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null} current state version: 4
   [junit4]   2> 450647 T825 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:51163_p_myp",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51163/p_myp",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 450648 T825 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 450648 T825 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 450649 T851 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 450649 T836 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 450649 T824 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 451646 T852 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 451647 T852 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 451648 T852 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 451648 T852 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 451649 T852 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 451649 T852 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001/cores/collection1/'
   [junit4]   2> 451666 T852 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 451669 T852 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 451680 T852 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 451702 T852 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 451752 T852 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 451853 T852 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 451865 T852 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 452307 T852 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 452325 T852 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 452329 T852 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 452340 T852 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 452345 T852 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 452350 T852 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 452351 T852 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 452351 T852 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 452351 T852 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 452352 T852 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 452352 T852 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 452353 T852 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 452353 T852 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 452353 T852 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 452354 T852 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@78a8ee09
   [junit4]   2> 452354 T852 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001/cores/collection1/data
   [junit4]   2> 452355 T852 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 452355 T852 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 452355 T852 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 94BF5CA3001FF936-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 452356 T852 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=18, maxMergeAtOnceExplicit=29, maxMergedSegmentMB=19.7001953125, floorSegmentMB=1.7607421875, forceMergeDeletesPctAllowed=20.3593349700404, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8547633598335697
   [junit4]   2> 452357 T852 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4fd319a7 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2c394be2),segFN=segments_1,generation=1}
   [junit4]   2> 452357 T852 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 452365 T852 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 452365 T852 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 452365 T852 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 452366 T852 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 452366 T852 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 452366 T852 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 452367 T852 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 452367 T852 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 452367 T852 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 452395 T852 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 452398 T852 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 452400 T852 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 452403 T852 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 452420 T852 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update/json/docs,/get,/admin/properties,/config,/admin/logging,/admin/threads,standard,/admin/system,/admin/ping,/admin/segments,/admin/plugins,/update/csv,/admin/file,/admin/mbeans,/schema,/update,/admin/luke,/replication,/update/json
   [junit4]   2> 452422 T852 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 452424 T852 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 452425 T852 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 452427 T852 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 452427 T852 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 452428 T852 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1118247169, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 452429 T852 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4fd319a7 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2c394be2),segFN=segments_1,generation=1}
   [junit4]   2> 452429 T852 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 452430 T852 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@4c479bf7[collection1] main
   [junit4]   2> 452430 T852 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 452431 T852 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 452431 T852 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 452432 T852 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 452432 T852 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 452432 T852 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 452433 T852 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 452433 T852 C:collection1 c:collection1 

[...truncated too long message...]

r/live_nodes
   [junit4]   2> 		4	/solr/overseer/queue
   [junit4]   2> 		4	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1529366 T6482 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 10 connections to: http://127.0.0.1:57725/z/su
   [junit4]   2> 1529366 T6482 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:57731/z/su
   [junit4]   2> 1529367 T6482 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 1 connections to: http://127.0.0.1:57715/z/su
   [junit4]   2> 1529367 T6482 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:57741/z/su
   [junit4]   2> 1529368 T6482 C:control_collection S:shard1 c:collection1 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 1529368 T6482 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:57725/z/su
   [junit4]   2> 1529369 T6482 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:57731/z/su
   [junit4]   2> 1529369 T6482 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:57715/z/su
   [junit4]   2> 1529369 T6482 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:57741/z/su
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=94BF5CA3001FF936 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=de_AT -Dtests.timezone=America/St_Barthelemy -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   78.4s J0 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:57725/z/su/repfacttest_c8n_1x3_shard1_replica1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([94BF5CA3001FF936:1CEB6379AEE394CE]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:57725/z/su/repfacttest_c8n_1x3_shard1_replica1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 48 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 53 more
   [junit4]   2> 1529401 T6482 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 94BF5CA3001FF936-001
   [junit4]   2> NOTE: test params are: codec=Lucene50, sim=DefaultSimilarity, locale=de_AT, timezone=America/St_Barthelemy
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=158683704,total=327155712
   [junit4]   2> NOTE: All tests run in this JVM: [TestConfigOverlay, TestEmbeddedSolrServerConstructors, SignatureUpdateProcessorFactoryTest, DistributedIntervalFacetingTest, DefaultValueUpdateProcessorTest, MoreLikeThisHandlerTest, HdfsUnloadDistributedZkTest, TestPostingsSolrHighlighter, TestSolrConfigHandlerConcurrent, AtomicUpdatesTest, TestZkChroot, TestFuzzyAnalyzedSuggestions, TestLFUCache, EchoParamsTest, BlockCacheTest, TermVectorComponentTest, HardAutoCommitTest, TestBinaryResponseWriter, ClusterStateTest, PeerSyncTest, TestJettySolrRunner, ExternalFileFieldSortTest, TestReRankQParserPlugin, IndexBasedSpellCheckerTest, SpatialFilterTest, TestSuggestSpellingConverter, DateFieldTest, TestWordDelimiterFilterFactory, TestCollationFieldDocValues, TestSolrIndexConfig, TestPivotHelperCode, TestIntervalFaceting, TestRTGBase, OverseerStatusTest, SolrIndexSplitterTest, WordBreakSolrSpellCheckerTest, TestSolrDeletionPolicy1, TestSolrCoreProperties, PluginInfoTest, TimeZoneUtilsTest, TestDistribDocBasedVersion, InfoHandlerTest, DistributedQueueTest, ZkStateWriterTest, TestDynamicFieldCollectionResource, TestLMJelinekMercerSimilarityFactory, ShowFileRequestHandlerTest, HdfsRecoveryZkTest, ShardRoutingTest, DistributedFacetPivotWhiteBoxTest, SortByFunctionTest, TestGroupingSearch, TestRangeQuery, TestIndexSearcher, DistributedFacetPivotSmallAdvancedTest, TestMiniSolrCloudCluster, TestTrackingShardHandlerFactory, HttpPartitionTest, ShardSplitTest, TestSolrConfigHandlerCloud, ReplicationFactorTest]
   [junit4] Completed on J0 in 78.39s, 1 test, 1 error <<< FAILURES!

[...truncated 697 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 4 errors, 61 ignored (21 assumptions)

Total time: 52 minutes 44 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 228586426 bytes
Compression is 0.0%
Took 42 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2878 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2878/

4 tests failed.
REGRESSION:  org.apache.solr.handler.component.DistributedMLTComponentTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:50578/r/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:50578/r/collection1
	at __randomizedtesting.SeedInfo.seed([7FD3AC6109AC3620:F78793BBA7505BD8]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:566)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:943)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:558)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:606)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:588)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:567)
	at org.apache.solr.handler.component.DistributedMLTComponentTest.test(DistributedMLTComponentTest.java:126)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 51 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:37228/_bog/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37228/_bog/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([7FD3AC6109AC3620:F78793BBA7505BD8]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([7FD3AC6109AC3620:F78793BBA7505BD8]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([7FD3AC6109AC3620:F78793BBA7505BD8]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9257 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/init-core-data-001
   [junit4]   2> 112850 T215 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /ebfhh/
   [junit4]   2> 112866 T215 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 112867 T216 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 112967 T215 oasc.ZkTestServer.run start zk server on port:36921
   [junit4]   2> 112968 T215 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 112970 T215 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 112976 T223 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5375b99f name:ZooKeeperConnection Watcher:127.0.0.1:36921 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 112977 T215 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 112978 T215 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 112978 T215 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 112983 T215 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 112984 T215 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 112987 T226 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@33d40863 name:ZooKeeperConnection Watcher:127.0.0.1:36921/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 112987 T215 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 112989 T215 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 112989 T215 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 112992 T215 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 112995 T215 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 112997 T215 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 113000 T215 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 113001 T215 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 113005 T215 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 113006 T215 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 113010 T215 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 113011 T215 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 113014 T215 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 113015 T215 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 113017 T215 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 113018 T215 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 113021 T215 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 113022 T215 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 113025 T215 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 113026 T215 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 113029 T215 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 113030 T215 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 113032 T215 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 113033 T215 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 113035 T215 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 113036 T215 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 113039 T215 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 113040 T215 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 115068 T215 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/control-001/cores/collection1
   [junit4]   2> 115074 T215 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 115080 T215 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:31943
   [junit4]   2> 115081 T215 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/ebfhh, hostPort=16926, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/control-001/cores}
   [junit4]   2> 115083 T215 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@5fb271f1
   [junit4]   2> 115084 T215 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/control-001/'
   [junit4]   2> 115168 T215 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/control-001/solr.xml
   [junit4]   2> 115291 T215 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/control-001/cores
   [junit4]   2> 115292 T215 oasc.CoreContainer.<init> New CoreContainer 1738635257
   [junit4]   2> 115293 T215 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/control-001/]
   [junit4]   2> 115293 T215 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/control-001/lib
   [junit4]   2> 115295 T215 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/control-001/lib).
   [junit4]   2> 115314 T215 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 115315 T215 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 115316 T215 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 115316 T215 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 115317 T215 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 115318 T215 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 115318 T215 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 115319 T215 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 115319 T215 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 115320 T215 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 115320 T215 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 115321 T215 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 115321 T215 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 115322 T215 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 115322 T215 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 115323 T215 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:36921/solr
   [junit4]   2> 115324 T215 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 115325 T215 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 115326 T215 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 115329 T238 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3533cc3 name:ZooKeeperConnection Watcher:127.0.0.1:36921 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 115330 T215 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 115331 T215 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 115336 T215 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 115338 T241 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6e692fe name:ZooKeeperConnection Watcher:127.0.0.1:36921/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 115339 T215 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 115342 T215 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 115346 T215 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 115349 T215 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 115352 T215 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 115355 T215 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 115358 T215 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 115361 T215 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 115363 T215 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 115365 T215 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:16926_ebfhh
   [junit4]   2> 115366 T215 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:16926_ebfhh
   [junit4]   2> 115369 T215 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 115371 T215 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 115373 T215 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 115375 T215 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:16926_ebfhh
   [junit4]   2> 115376 T215 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 115379 T215 oasc.Overseer.start Overseer (id=93585283107192835-127.0.0.1:16926_ebfhh-n_0000000000) starting
   [junit4]   2> 115382 T215 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 115391 T215 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 115393 T243 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 115393 T215 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 115394 T242 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 115400 T215 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/control-001/cores
   [junit4]   2> 115404 T215 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {coreNodeName=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/control-001/cores/collection1, dataDir=data/, shard=, schema=schema.xml, loadOnStartup=true, transient=false, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/control-001/cores/collection1/, collection=control_collection, config=solrconfig.xml, name=collection1}
   [junit4]   2> 115404 T215 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/control-001/cores/collection1/
   [junit4]   2> 115405 T215 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 115407 T245 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 115408 T245 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 115409 T245 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 115409 T241 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 115425 T242 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:16926/ebfhh",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:16926_ebfhh",
   [junit4]   2> 	  "operation":"state"} current state version: 0
   [junit4]   2> 115426 T242 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:16926/ebfhh",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:16926_ebfhh",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 115429 T242 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 115430 T242 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 115434 T241 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 116410 T245 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 116411 T245 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 116412 T245 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 116412 T245 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 116413 T245 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 116414 T245 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/control-001/cores/collection1/'
   [junit4]   2> 116436 T245 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 116439 T245 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 116456 T245 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 116483 T245 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 116555 T245 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 116557 T245 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 116586 T245 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 117147 T245 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 117170 T245 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 117174 T245 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 117200 T245 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 117260 T245 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 117268 T245 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 117269 T245 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 117270 T245 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 117270 T245 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 117272 T245 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 117272 T245 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 117272 T245 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 117273 T245 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 117273 T245 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/control-001/cores/collection1/data/
   [junit4]   2> 117274 T245 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4e965d5
   [junit4]   2> 117275 T245 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/control-001/cores/collection1/data
   [junit4]   2> 117275 T245 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/control-001/cores/collection1/data/index/
   [junit4]   2> 117276 T245 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 117276 T245 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/control-001/cores/collection1/data/index
   [junit4]   2> 117277 T245 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=22, maxMergedSegmentMB=84.2001953125, floorSegmentMB=0.2041015625, forceMergeDeletesPctAllowed=22.20860105460578, segmentsPerTier=29.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 117278 T245 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@21ae0eca lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2a4d010c),segFN=segments_1,generation=1}
   [junit4]   2> 117279 T245 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 117289 T245 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 117290 T245 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 117290 T245 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 117291 T245 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 117291 T245 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 117292 T245 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 117292 T245 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 117293 T245 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 117293 T245 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 117316 T245 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 117320 T245 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 117323 T245 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 117326 T245 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 117334 T245 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/luke,standard,/config,/admin/mbeans,/admin/segments,/admin/logging,/get,/admin/file,/admin/plugins,/admin/ping,/admin/system,/schema,/admin/properties,/update/json,/update/json/docs,/replication,/admin/threads,/update/csv,/update
   [junit4]   2> 117338 T245 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 117341 T245 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 117342 T245 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 117344 T245 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 117345 T245 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 117347 T245 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1200085076, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 117348 T245 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@21ae0eca lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2a4d010c),segFN=segments_1,generation=1}
   [junit4]   2> 117349 T245 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 117350 T245 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@35a0c598[collection1] main
   [junit4]   2> 117351 T245 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 117352 T245 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 117353 T245 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 117353 T245 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 117354 T245 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 117354 T245 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 117355 T245 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 117356 T245 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 117356 T245 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 117356 T245 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 117358 T246 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@35a0c598[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 117359 T245 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 117360 T249 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:16926/ebfhh collection:control_collection shard:shard1
   [junit4]   2> 117360 T215 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 117361 T215 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 117361 T249 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 117365 T215 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 117367 T215 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 117369 T253 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4a60166d name:ZooKeeperConnection Watcher:127.0.0.1:36921/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 117369 T249 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 117369 T215 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 117371 T215 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 117371 T215 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 117372 T241 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 117372 T249 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 117373 T249 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 117373 T242 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C60 name=collection1 org.apache.solr.core.SolrCore@34c927df url=http://127.0.0.1:16926/ebfhh/collection1 node=127.0.0.1:16926_ebfhh C60_STATE=coll:control_collection core:collection1 props:{state=down, core=collection1, base_url=http://127.0.0.1:16926/ebfhh, node_name=127.0.0.1:16926_ebfhh}
   [junit4]   2> 117373 T249 C:control_collection S:shard1 C60 P16926 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:16926/ebfhh/collection1/
   [junit4]   2> 117374 T249 C:control_collection S:shard1 C60 P16926 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 117375 T249 C:control_collection S:shard1 C60 P16926 oasc.SyncStrategy.syncToMe http://127.0.0.1:16926/ebfhh/collection1/ has no replicas
   [junit4]   2> 117375 T215 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 117376 T253 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 117376 T241 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 117376 T249 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:16926/ebfhh/collection1/ shard1
   [junit4]   2> 117376 T215 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 117379 T249 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 117379 T215 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 117381 T215 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 117383 T256 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@623c3ef0 name:ZooKeeperConnection Watcher:127.0.0.1:36921/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 117384 T215 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 117385 T215 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 117386 T241 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 117387 T242 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:16926/ebfhh",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 117392 T242 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 117393 T242 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 117394 T253 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 117394 T241 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 117399 T241 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 117400 T242 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 117402 T253 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 117402 T241 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 117437 T249 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:16926/ebfhh/collection1/ and leader is http://127.0.0.1:16926/ebfhh/collection1/
   [junit4]   2> 117438 T249 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:16926/ebfhh
   [junit4]   2> 117439 T249 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 117439 T249 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 117440 T249 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 117442 T241 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 117446 T242 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:16926/ebfhh",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:16926_ebfhh",
   [junit4]   2> 	  "operation":"state"} current state version: 4
   [junit4]   2> 117447 T242 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:16926/ebfhh",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:16926_ebfhh",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 117553 T253 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 117553 T241 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 119030 T215 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001/cores/collection1
   [junit4]   2> 119034 T215 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001
   [junit4]   2> 119035 T215 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 119038 T215 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:60867
   [junit4]   2> 119039 T215 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/ebfhh, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/tempDir-001/jetty1, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001/cores, hostPort=52711, solrconfig=solrconfig.xml}
   [junit4]   2> 119040 T215 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@5fb271f1
   [junit4]   2> 119040 T215 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001/'
   [junit4]   2> 119098 T215 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001/solr.xml
   [junit4]   2> 119164 T215 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001/cores
   [junit4]   2> 119165 T215 oasc.CoreContainer.<init> New CoreContainer 383362931
   [junit4]   2> 119165 T215 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001/]
   [junit4]   2> 119165 T215 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001/lib
   [junit4]   2> 119166 T215 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001/lib).
   [junit4]   2> 119181 T215 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 119181 T215 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 119182 T215 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 119183 T215 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 119183 T215 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 119184 T215 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 119184 T215 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 119184 T215 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 119185 T215 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 119185 T215 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 119186 T215 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 119187 T215 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 119187 T215 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 119188 T215 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 119188 T215 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 119188 T215 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:36921/solr
   [junit4]   2> 119189 T215 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 119189 T215 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 119191 T215 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 119198 T268 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7191b8be name:ZooKeeperConnection Watcher:127.0.0.1:36921 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 119199 T215 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 119200 T215 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 119203 T215 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 119205 T271 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@617ceddc name:ZooKeeperConnection Watcher:127.0.0.1:36921/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 119205 T215 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 119211 T215 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 120218 T215 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52711_ebfhh
   [junit4]   2> 120219 T215 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:52711_ebfhh
   [junit4]   2> 120223 T215 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 120226 T215 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001/cores
   [junit4]   2> 120230 T215 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001/cores/collection1, schema=schema.xml, name=collection1, shard=, transient=false, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001/cores/collection1/, dataDir=data/, loadOnStartup=true, config=solrconfig.xml, coreNodeName=}
   [junit4]   2> 120230 T215 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001/cores/collection1/
   [junit4]   2> 120231 T215 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 120234 T272 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 120235 T272 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 120237 T241 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 120238 T272 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 120238 T272 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 120238 T242 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:52711/ebfhh",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:52711_ebfhh",
   [junit4]   2> 	  "operation":"state"} current state version: 5
   [junit4]   2> 120239 T272 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 120239 T242 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:52711/ebfhh",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:52711_ebfhh",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 120240 T242 oasco.ReplicaMutator.updateState Collection already exists with numShards=3
   [junit4]   2> 120240 T242 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard3
   [junit4]   2> 120240 T272 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 120241 T272 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 120243 T242 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 120245 T271 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 120245 T271 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 121241 T272 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 121242 T272 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 121243 T272 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 121243 T272 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 121244 T272 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 121245 T272 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001/cores/collection1/'
   [junit4]   2> 121264 T272 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 121267 T272 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 121283 T272 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 121313 T272 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 121380 T272 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 121382 T272 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 121405 T272 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 121938 T272 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 121959 T272 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 121964 T272 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 121989 T272 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 121997 T272 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 122003 T272 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 122004 T272 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 122004 T272 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 122005 T272 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 122006 T272 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 122006 T272 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 122007 T272 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 122007 T272 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 122007 T272 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 122008 T272 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4e965d5
   [junit4]   2> 122010 T272 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001/cores/collection1/data
   [junit4]   2> 122010 T272 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 122010 T272 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 122012 T272 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7FD3AC6109AC3620-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 122012 T272 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=22, maxMergedSegmentMB=84.2001953125, floorSegmentMB=0.2041015625, forceMergeDeletesPctAllowed=22.20860105460578, segmentsPerTier=29.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 122013 T272 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@70817d51 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@454a8a8b),segFN=segments_1,generation=1}
   [junit4]   2> 122014 T272 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 122022 T272 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 122023 T272 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 122023 T272 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 122024 T272 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 122024 T272 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 122024 T272 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 122025 T272 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 122025 T272 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 122025 T272 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 122039 T272 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 122041 T272 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 122043 T272 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 122046 T272 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 122053 T272 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/luke,standard,/config,/admin/mbeans,/admin/segments,/admin/logging,/get,/admin/file,/admin/plugins,/admin/ping,/admin/system,/schema,/admin/properties,/update/json,/update/json/docs,/replication,/admin/threads,/update/csv,/update
   [junit4]   2> 122056 T272 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 122057 T272 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 122057 T272 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 122059 T272 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 122059 T272 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 122061 T272 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1200085076, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 122062 T272 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@70817d51 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4

[...truncated too long message...]

aliases.json
   [junit4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 		4	/solr/collections/collection1/state.json
   [junit4]   2> 		3	/solr/collections/c8n_1x3_commits/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent data watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		2	/solr/collections/c8n_1x3_commits/leader_elect/shard1/election/93585371076821001-core_node1-n_0000000000
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/queue
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1532959 T6241 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:37228/_bog
   [junit4]   2> 1532960 T6241 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:37254/_bog
   [junit4]   2> 1532960 T6241 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:37165/_bog
   [junit4]   2> 1532961 T6241 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:37209/_bog
   [junit4]   2> 1532961 T6241 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 11 connections to: http://127.0.0.1:37189/_bog
   [junit4]   2> 1532961 T6241 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:37228/_bog
   [junit4]   2> 1532962 T6241 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:37254/_bog
   [junit4]   2> 1532962 T6241 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:37165/_bog
   [junit4]   2> 1532962 T6241 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:37209/_bog
   [junit4]   2> 1532962 T6241 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:37189/_bog
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderInitiatedRecoveryOnCommitTest -Dtests.method=test -Dtests.seed=7FD3AC6109AC3620 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ar_MA -Dtests.timezone=America/Argentina/Rio_Gallegos -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   78.2s J2 | LeaderInitiatedRecoveryOnCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37228/_bog/c8n_1x3_commits_shard1_replica1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([7FD3AC6109AC3620:F78793BBA7505BD8]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 48 more
   [junit4]   2> 1532986 T6241 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7FD3AC6109AC3620-001
   [junit4]   2> 78159 T6240 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=true,coord=crazy): {}, locale=ar_MA, timezone=America/Argentina/Rio_Gallegos
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=120600672,total=369623040
   [junit4]   2> NOTE: All tests run in this JVM: [WordBreakSolrSpellCheckerTest, TestDistributedSearch, TestPerFieldSimilarity, TestHighlightDedupGrouping, SharedFSAutoReplicaFailoverTest, TestBadConfig, TestCharFilters, TestWriterPerf, TestSerializedLuceneMatchVersion, TestFieldTypeResource, TestSweetSpotSimilarityFactory, TestDistributedGrouping, TestCollapseQParserPlugin, TestMaxScoreQueryParser, SuggesterFSTTest, TestReRankQParserPlugin, TestLRUStatsCache, DeleteShardTest, TestLazyCores, MinimalSchemaTest, ScriptEngineTest, SolrCmdDistributorTest, TestCloudSchemaless, TestExactSharedStatsCache, AddSchemaFieldsUpdateProcessorFactoryTest, PeerSyncTest, TestCloudPivotFacet, TestFastWriter, TestFieldCollectionResource, SchemaVersionSpecificBehaviorTest, DisMaxRequestHandlerTest, SolrIndexSplitterTest, SpellCheckComponentTest, TestNonDefinedSimilarityFactory, DistributedFacetPivotWhiteBoxTest, TestMergePolicyConfig, IndexSchemaRuntimeFieldTest, TestRestManager, EnumFieldTest, QueryEqualityTest, TestInitQParser, DistribCursorPagingTest, TestExtendedDismaxParser, HdfsWriteToMultipleCollectionsTest, DeleteInactiveReplicaTest, CircularListTest, QueryElevationComponentTest, ZkControllerTest, TestRandomFaceting, TestSolrConfigHandlerCloud, TestUtils, TestBlendedInfixSuggestions, TestSolrDynamicMBean, TestShortCircuitedRequests, SyncSliceTest, TestFiltering, CurrencyFieldXmlFileTest, ShardSplitTest, LeaderInitiatedRecoveryOnCommitTest]
   [junit4] Completed on J2 in 78.70s, 1 test, 1 error <<< FAILURES!

[...truncated 711 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 4 errors, 61 ignored (21 assumptions)

Total time: 53 minutes 58 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 214333640 bytes
Compression is 0.0%
Took 38 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2877 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2877/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:50644/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:50644/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([EAB2019C6240E07D:62E63E46CCBC8D85]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:17642/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:17642/collection1
	at __randomizedtesting.SeedInfo.seed([EAB2019C6240E07D:62E63E46CCBC8D85]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:17642/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:18480/kl_/o/repfacttest_c8n_1x3_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:18480/kl_/o/repfacttest_c8n_1x3_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([EAB2019C6240E07D:62E63E46CCBC8D85]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:18480/kl_/o/repfacttest_c8n_1x3_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9318 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/init-core-data-001
   [junit4]   2> 131839 T341 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 131848 T341 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 131849 T342 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 131949 T341 oasc.ZkTestServer.run start zk server on port:17610
   [junit4]   2> 131953 T341 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 131954 T341 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 131958 T349 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3892a8c3 name:ZooKeeperConnection Watcher:127.0.0.1:17610 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 131958 T341 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 131959 T341 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 131959 T341 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 131964 T341 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 131965 T341 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 131967 T352 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@92e546b name:ZooKeeperConnection Watcher:127.0.0.1:17610/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 131967 T341 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 131968 T341 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 131968 T341 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 131973 T341 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 131977 T341 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 131979 T341 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 131984 T341 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 131984 T341 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 131990 T341 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 131991 T341 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 131996 T341 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 131997 T341 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 132001 T341 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 132002 T341 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 132006 T341 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 132007 T341 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 132011 T341 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 132012 T341 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 132016 T341 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 132016 T341 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 132020 T341 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 132021 T341 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 132025 T341 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 132026 T341 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 132030 T341 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 132030 T341 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 132034 T341 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 132035 T341 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 133733 T341 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/control-001/cores/collection1
   [junit4]   2> 133739 T341 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 133749 T341 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:17615
   [junit4]   2> 133749 T341 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/control-001/cores, hostPort=17614, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/tempDir-001/control/data}
   [junit4]   2> 133750 T341 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 133751 T341 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/control-001/'
   [junit4]   2> 133805 T341 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/control-001/solr.xml
   [junit4]   2> 133877 T341 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/control-001/cores
   [junit4]   2> 133877 T341 oasc.CoreContainer.<init> New CoreContainer 1810978688
   [junit4]   2> 133878 T341 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/control-001/]
   [junit4]   2> 133878 T341 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/control-001/lib
   [junit4]   2> 133879 T341 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/control-001/lib).
   [junit4]   2> 133894 T341 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 133895 T341 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 133895 T341 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 133895 T341 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 133896 T341 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 133896 T341 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 133897 T341 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 133897 T341 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 133897 T341 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 133898 T341 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 133898 T341 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 133899 T341 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 133900 T341 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 133900 T341 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 133900 T341 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 133901 T341 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:17610/solr
   [junit4]   2> 133901 T341 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 133902 T341 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 133903 T341 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 133905 T364 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@a8a4795 name:ZooKeeperConnection Watcher:127.0.0.1:17610 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 133906 T341 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 133907 T341 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 133910 T341 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 133912 T367 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1b778e41 name:ZooKeeperConnection Watcher:127.0.0.1:17610/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 133912 T341 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 133915 T341 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 133919 T341 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 133924 T341 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 133928 T341 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 133933 T341 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 133938 T341 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 133942 T341 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 133945 T341 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 133948 T341 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:17614_
   [junit4]   2> 133949 T341 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:17614_
   [junit4]   2> 133952 T341 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 133955 T341 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 133958 T341 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 133962 T341 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:17614_
   [junit4]   2> 133962 T341 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 133965 T341 oasc.Overseer.start Overseer (id=93583903181307907-127.0.0.1:17614_-n_0000000000) starting
   [junit4]   2> 133969 T341 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 133982 T341 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 133983 T369 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 133983 T341 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 133984 T368 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 133992 T341 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/control-001/cores
   [junit4]   2> 134007 T341 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=control_collection, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/control-001/cores/collection1/, dataDir=data/, shard=, config=solrconfig.xml, schema=schema.xml, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/control-001/cores/collection1, name=collection1, coreNodeName=, loadOnStartup=true}
   [junit4]   2> 134007 T341 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/control-001/cores/collection1/
   [junit4]   2> 134008 T341 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 134010 T371 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 134011 T371 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 134020 T367 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 134020 T371 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 134029 T368 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:17614",
   [junit4]   2> 	  "node_name":"127.0.0.1:17614_"} current state version: 0
   [junit4]   2> 134090 T368 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:17614",
   [junit4]   2> 	  "node_name":"127.0.0.1:17614_"}
   [junit4]   2> 134097 T368 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 134155 T368 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 134159 T367 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 135022 T371 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 135022 T371 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 135023 T371 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 135024 T371 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 135025 T371 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 135026 T371 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/control-001/cores/collection1/'
   [junit4]   2> 135089 T371 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 135120 T371 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 135163 T371 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 135198 T371 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 135261 T371 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 135264 T371 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 135283 T371 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 135799 T371 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 135824 T371 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 135828 T371 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 135861 T371 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 135869 T371 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 135876 T371 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 135878 T371 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 135878 T371 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 135879 T371 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 135881 T371 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 135881 T371 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 135882 T371 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 135882 T371 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 135883 T371 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/control-001/cores/collection1/data/
   [junit4]   2> 135883 T371 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@77d0bef0
   [junit4]   2> 135884 T371 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/control-001/cores/collection1/data
   [junit4]   2> 135885 T371 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/control-001/cores/collection1/data/index/
   [junit4]   2> 135885 T371 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 135886 T371 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/control-001/cores/collection1/data/index
   [junit4]   2> 135886 T371 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=20, maxMergeAtOnceExplicit=28, maxMergedSegmentMB=64.7998046875, floorSegmentMB=0.791015625, forceMergeDeletesPctAllowed=25.56615195121221, segmentsPerTier=37.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 135887 T371 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@29ee6f9e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@64e20393),segFN=segments_1,generation=1}
   [junit4]   2> 135888 T371 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 135897 T371 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 135987 T371 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 135988 T371 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 135988 T371 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 135988 T371 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 135989 T371 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 135989 T371 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 135990 T371 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 135990 T371 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 136013 T371 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 136016 T371 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 136018 T371 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 136021 T371 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 136028 T371 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: standard,/admin/properties,/admin/segments,/admin/threads,/update/csv,/admin/system,/update,/replication,/admin/plugins,/admin/luke,/schema,/admin/file,/admin/mbeans,/admin/logging,/admin/ping,/config,/get,/update/json/docs,/update/json
   [junit4]   2> 136031 T371 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 136033 T371 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 136033 T371 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 136036 T371 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 136036 T371 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 136037 T371 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=19, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 136038 T371 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@29ee6f9e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@64e20393),segFN=segments_1,generation=1}
   [junit4]   2> 136038 T371 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 136039 T371 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@726aef5c[collection1] main
   [junit4]   2> 136039 T371 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 136041 T371 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 136041 T371 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 136056 T371 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 136056 T371 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 136057 T371 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 136058 T371 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 136058 T371 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 136059 T371 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 136059 T371 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 136060 T372 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@726aef5c[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 136068 T371 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 136070 T375 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:17614 collection:control_collection shard:shard1
   [junit4]   2> 136070 T341 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 136071 T341 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 136086 T341 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 136087 T341 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 136090 T379 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7940bbc5 name:ZooKeeperConnection Watcher:127.0.0.1:17610/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 136090 T341 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 136092 T341 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 136093 T341 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 136102 T341 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 136103 T341 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 136103 T341 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 136104 T341 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 136106 T382 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@521f3cfb name:ZooKeeperConnection Watcher:127.0.0.1:17610/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 136107 T341 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 136107 T341 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 136109 T375 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 136111 T367 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 136114 T368 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 1
   [junit4]   2> 136114 T368 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 136118 T368 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 136120 T375 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 136121 T367 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 136121 T379 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 136125 T375 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 136125 T375 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 136130 T368 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 2
   [junit4]   2> ASYNC  NEW_CORE C25 name=collection1 org.apache.solr.core.SolrCore@321ec652 url=http://127.0.0.1:17614/collection1 node=127.0.0.1:17614_ C25_STATE=coll:control_collection core:collection1 props:{core=collection1, state=down, base_url=http://127.0.0.1:17614, node_name=127.0.0.1:17614_}
   [junit4]   2> 136139 T375 C:control_collection S:shard1 C25 P17614 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:17614/collection1/
   [junit4]   2> 136140 T375 C:control_collection S:shard1 C25 P17614 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 136140 T375 C:control_collection S:shard1 C25 P17614 oasc.SyncStrategy.syncToMe http://127.0.0.1:17614/collection1/ has no replicas
   [junit4]   2> 136140 T375 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:17614/collection1/ shard1
   [junit4]   2> 136150 T367 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 136153 T379 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 136153 T367 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 136157 T375 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 136167 T367 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 136169 T368 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:17614",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 3
   [junit4]   2> 136276 T379 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 136276 T367 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 136322 T375 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:17614/collection1/ and leader is http://127.0.0.1:17614/collection1/
   [junit4]   2> 136322 T375 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:17614
   [junit4]   2> 136322 T375 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 136323 T375 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 136323 T375 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 136326 T367 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 136328 T368 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:17614",
   [junit4]   2> 	  "node_name":"127.0.0.1:17614_"} current state version: 4
   [junit4]   2> 136330 T368 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:17614",
   [junit4]   2> 	  "node_name":"127.0.0.1:17614_"}
   [junit4]   2> 136437 T367 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 136437 T379 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 137799 T341 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001/cores/collection1
   [junit4]   2> 137803 T341 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001
   [junit4]   2> 137804 T341 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 137807 T341 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:17643
   [junit4]   2> 137808 T341 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solrconfig=solrconfig.xml, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001/cores, hostContext=/, hostPort=17642, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/tempDir-001/jetty1}
   [junit4]   2> 137809 T341 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 137809 T341 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001/'
   [junit4]   2> 137864 T341 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001/solr.xml
   [junit4]   2> 137931 T341 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001/cores
   [junit4]   2> 137932 T341 oasc.CoreContainer.<init> New CoreContainer 1978325
   [junit4]   2> 137933 T341 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001/]
   [junit4]   2> 137933 T341 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001/lib
   [junit4]   2> 137934 T341 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001/lib).
   [junit4]   2> 137950 T341 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 137951 T341 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 137951 T341 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 137952 T341 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 137952 T341 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 137952 T341 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 137953 T341 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 137953 T341 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 137954 T341 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 137954 T341 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 137954 T341 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 137955 T341 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 137956 T341 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 137956 T341 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 137957 T341 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 137957 T341 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:17610/solr
   [junit4]   2> 137957 T341 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 137958 T341 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 137959 T341 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 137961 T394 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4c0af79c name:ZooKeeperConnection Watcher:127.0.0.1:17610 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 137962 T341 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 137963 T341 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 137966 T341 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 137969 T397 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2a87393d name:ZooKeeperConnection Watcher:127.0.0.1:17610/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 137969 T341 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 137979 T341 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 138991 T341 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:17642_
   [junit4]   2> 138992 T341 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:17642_
   [junit4]   2> 138997 T341 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 139002 T341 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001/cores
   [junit4]   2> 139005 T341 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {loadOnStartup=true, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001/cores/collection1, config=solrconfig.xml, shard=, coreNodeName=, dataDir=data/, transient=false, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001/cores/collection1/, collection=collection1, schema=schema.xml, name=collection1}
   [junit4]   2> 139005 T341 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001/cores/collection1/
   [junit4]   2> 139006 T341 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 139010 T398 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 139010 T398 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 139013 T367 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 139015 T398 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 139015 T368 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:17642",
   [junit4]   2> 	  "node_name":"127.0.0.1:17642_"} current state version: 5
   [junit4]   2> 139015 T398 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 139017 T368 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:17642",
   [junit4]   2> 	  "node_name":"127.0.0.1:17642_"}
   [junit4]   2> 139017 T398 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 139017 T368 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 139018 T368 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 139021 T368 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 139029 T398 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 139030 T398 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 139030 T398 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 139030 T398 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 139031 T398 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 139031 T398 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 139033 T398 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 139033 T398 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001/cores/collection1/'
   [junit4]   2> 139052 T398 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 139056 T398 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 139071 T398 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 139096 T398 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 139155 T398 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 139157 T398 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 139177 T398 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 139676 T398 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 139698 T398 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 139701 T398 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 139730 T398 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 139736 T398 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 139742 T398 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 139744 T398 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 139744 T398 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 139745 T398 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 139746 T398 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 139746 T398 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 139747 T398 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 139747 T398 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 139748 T398 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 139748 T398 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@77d0bef0
   [junit4]   2> 139749 T398 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001/cores/collection1/data
   [junit4]   2> 139749 T398 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 139750 T398 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 139750 T398 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest EAB2019C6240E07D-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 139751 T398 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=20, maxMergeAtOnceExplicit=28, maxMergedSegmentMB=64.7998046875, floorSegmentMB=0.791015625, forceMergeDeletesPctAllowed=25.56615195121221, segmentsPerTier=37.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 139752 T398 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1edb0c6d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3f0cd7ee),segFN=segments_1,generation=1}
   [junit4]   2> 139752 T398 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 139763 T398 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 139764 T398 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 139764 T398 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 139765 T398 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 139765 T398 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 139766 T398 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 139766 T398 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 139767 T398 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 139767 T398 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 139788 T398 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 139791 T398 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 139794 T398 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 139796 T398 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 139837 T398 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: standard,/admin/properties,/admin/segments,/admin/threads,/update/csv,/admin/system,/update,/replication,/admin/plugins,/admin/luke,/schema,/admin/file,/admin/mbeans,/admin/logging,/admin/ping,/config,/get,/update/json/docs,/update/json
   [junit4]   2> 139840 T398 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 139842 T398 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 139842 T398 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 139844 T398 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 139844 T398 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 139845 T398 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=19, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 139846 T398 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1edb0c6d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3f0cd7ee),segFN=segments_1,generation=1}
   [junit4]   2> 139847 T398 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 139847 T398 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@5aba3e7f[collection1] main
   [junit4]   2> 139847 T398 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 139849 T398 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 139850 T398 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 139850 T398 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 139851 T398 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 139851 T398 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 139852 T398 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 139852 T398 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 139852 T398 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 139853 T398 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 139854 T399 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5aba3e7f[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 139854 T398 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 139855 T402 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:17642 collection:collection1 shard:shard1
   [junit4]   2> 139856 T341 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 139857 T341 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 139857 T402 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 139864 T402 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 139867 T367 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 139867 T402 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 139868 T402 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C26 name=collection1 org.apache.solr.core.SolrCore@46980bbf url=http://127.0.0.1:17642/collection1 node=127.0.0.1:17642_ C26_STATE=coll:collection1 core:collection1 props:{core=collection1, state=down, base_url=http://127.0.0.1:17642, node_name=127.0.0.1:17642_}
   [junit4]   2> 139868 T402 C:collection1 S:shard1 C26 P17642 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:17642/collection1/
   [junit4]   2> 139868 T402 C:collection1 S:shard1 C26 P17642 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 139868 T368 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 139868 T402 C:collection1 S:shard1 C26 P17642 oasc.SyncStrategy.syncToMe http://127.0.0.1:17642/collection1/ has no replicas
   [junit4]   2> 139869 T402 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:17642/collection1/ shard1
   [junit4]   2> 139869 T402 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 139871 T368 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 139873 T397 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 139874 T397 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 139878 T367 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 139879 T368 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:17642",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 139882 T368 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 139883 T397 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 139884 T397 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 139930 T402 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:17642/collection1/ and leader is http://127.0.0.1:17642/collection1/
   [junit4]   2> 139931 T402 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:17642
   [junit4]   2> 139931 T402 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 139931 T402 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 139932 T402 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 139934 T367 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 139936 T368 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = 

[...truncated too long message...]

e.json
   [junit4]   2> 		6	/solr/aliases.json
   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 		4	/solr/collections/collection1/state.json
   [junit4]   2> 		3	/solr/collections/c8n_1x3_commits/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent data watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		2	/solr/collections/c8n_1x3_commits/leader_elect/shard1/election/93583987909263373-core_node3-n_0000000000
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 		4	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1503904 T7084 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:50662/
   [junit4]   2> 1503904 T7084 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:50644/
   [junit4]   2> 1503904 T7084 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:40631/
   [junit4]   2> 1503904 T7084 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 10 connections to: http://127.0.0.1:50635/
   [junit4]   2> 1503904 T7084 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 3 connections to: http://127.0.0.1:50625/
   [junit4]   2> 1503905 T7084 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:50662/
   [junit4]   2> 1503905 T7084 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:50644/
   [junit4]   2> 1503905 T7084 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:40631/
   [junit4]   2> 1503905 T7084 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:50635/
   [junit4]   2> 1503906 T7084 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:50625/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderInitiatedRecoveryOnCommitTest -Dtests.method=test -Dtests.seed=EAB2019C6240E07D -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=hi_IN -Dtests.timezone=Australia/Eucla -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   79.7s J2 | LeaderInitiatedRecoveryOnCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:50644/c8n_1x3_commits_shard1_replica2
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([EAB2019C6240E07D:62E63E46CCBC8D85]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 48 more
   [junit4]   2> 1503926 T7084 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest EAB2019C6240E07D-001
   [junit4]   2> 79740 T7083 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=DefaultSimilarity, locale=hi_IN, timezone=Australia/Eucla
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=151235736,total=389545984
   [junit4]   2> NOTE: All tests run in this JVM: [TestCryptoKeys, ScriptEngineTest, SolrTestCaseJ4Test, ShardSplitTest, OpenExchangeRatesOrgProviderTest, TestMiniSolrCloudCluster, DocValuesMissingTest, CurrencyFieldOpenExchangeTest, DocValuesTest, TestJmxMonitoredMap, DistribDocExpirationUpdateProcessorTest, TestShardHandlerFactory, TestBulkSchemaConcurrent, DocExpirationUpdateProcessorFactoryTest, CachingDirectoryFactoryTest, StatelessScriptUpdateProcessorFactoryTest, DistanceFunctionTest, TestReplicaProperties, TestFileDictionaryLookup, SolrXmlInZkTest, TestExactStatsCache, SegmentsInfoRequestHandlerTest, TestQuerySenderNoQuery, TestLRUCache, RequestHandlersTest, DirectSolrConnectionTest, CoreMergeIndexesAdminHandlerTest, TestClusterStateMutator, TestInfoStreamLogging, TestIBSimilarityFactory, DistributedQueueTest, ReturnFieldsTest, CSVRequestHandlerTest, SimpleFacetsTest, TestRestManager, ClusterStateUpdateTest, TestXIncludeConfig, TestLeaderElectionZkExpiry, RAMDirectoryFactoryTest, TestUniqueKeyFieldResource, CircularListTest, FacetPivotSmallTest, TestReload, BasicZkTest, TestPerFieldSimilarity, TestRebalanceLeaders, OverseerCollectionProcessorTest, TestDistributedSearch, DefaultValueUpdateProcessorTest, TestReRankQParserPlugin, TestCustomSort, AlternateDirectoryTest, PrimUtilsTest, TestSolrDynamicMBean, SchemaVersionSpecificBehaviorTest, ExternalCollectionsTest, SolrIndexSplitterTest, ClusterStateTest, TestCloudSchemaless, TestCloudManagedSchemaConcurrent, ChaosMonkeyNothingIsSafeTest, UnloadDistributedZkTest, LeaderInitiatedRecoveryOnCommitTest]
   [junit4] Completed on J2 in 80.02s, 1 test, 1 error <<< FAILURES!

[...truncated 714 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 53 minutes 1 second
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 239090085 bytes
Compression is 0.0%
Took 44 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2876 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2876/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:24846/hff/p/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:24846/hff/p/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([177722754B01BEA:89234DFDFA4C7612]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([177722754B01BEA:89234DFDFA4C7612]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([177722754B01BEA:89234DFDFA4C7612]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9763 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/init-core-data-001
   [junit4]   2> 749957 T1332 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /_ww/ab
   [junit4]   2> 749969 T1332 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 749970 T1333 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 750070 T1332 oasc.ZkTestServer.run start zk server on port:25128
   [junit4]   2> 750071 T1332 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 750072 T1332 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 750076 T1340 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5001b139 name:ZooKeeperConnection Watcher:127.0.0.1:25128 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 750077 T1332 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 750077 T1332 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 750078 T1332 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 750080 T1332 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 750081 T1332 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 750083 T1343 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3ec5b62d name:ZooKeeperConnection Watcher:127.0.0.1:25128/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 750084 T1332 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 750084 T1332 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 750085 T1332 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 750087 T1332 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 750089 T1332 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 750091 T1332 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 750093 T1332 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 750094 T1332 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 750096 T1332 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 750097 T1332 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 750099 T1332 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 750100 T1332 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 750102 T1332 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 750102 T1332 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 750104 T1332 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 750105 T1332 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 750107 T1332 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 750107 T1332 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 750110 T1332 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 750110 T1332 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 750112 T1332 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 750112 T1332 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 750114 T1332 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 750115 T1332 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 750116 T1332 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 750117 T1332 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 750119 T1332 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 750119 T1332 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 751484 T1332 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/control-001/cores/collection1
   [junit4]   2> 751488 T1332 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 751494 T1332 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:25136
   [junit4]   2> 751494 T1332 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=25135, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/control-001/cores, hostContext=/_ww/ab, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/tempDir-001/control/data}
   [junit4]   2> 751495 T1332 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 751496 T1332 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/control-001/'
   [junit4]   2> 751541 T1332 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/control-001/solr.xml
   [junit4]   2> 751588 T1332 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/control-001/cores
   [junit4]   2> 751589 T1332 oasc.CoreContainer.<init> New CoreContainer 544904992
   [junit4]   2> 751589 T1332 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/control-001/]
   [junit4]   2> 751589 T1332 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/control-001/lib
   [junit4]   2> 751590 T1332 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/control-001/lib).
   [junit4]   2> 751605 T1332 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 751606 T1332 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 751606 T1332 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 751606 T1332 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 751606 T1332 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 751607 T1332 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 751607 T1332 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 751607 T1332 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 751607 T1332 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 751608 T1332 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 751608 T1332 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 751608 T1332 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 751609 T1332 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 751609 T1332 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 751609 T1332 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 751609 T1332 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:25128/solr
   [junit4]   2> 751610 T1332 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 751610 T1332 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 751611 T1332 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 751613 T1355 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@68cbe092 name:ZooKeeperConnection Watcher:127.0.0.1:25128 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 751614 T1332 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 751614 T1332 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 751617 T1332 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 751619 T1358 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1e8b171f name:ZooKeeperConnection Watcher:127.0.0.1:25128/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 751619 T1332 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 751621 T1332 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 751623 T1332 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 751625 T1332 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 751626 T1332 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 751628 T1332 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 751630 T1332 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 751631 T1332 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 751632 T1332 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 751634 T1332 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:25135__ww%2Fab
   [junit4]   2> 751634 T1332 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:25135__ww%2Fab
   [junit4]   2> 751636 T1332 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 751637 T1332 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 751638 T1332 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 751640 T1332 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:25135__ww%2Fab
   [junit4]   2> 751640 T1332 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 751641 T1332 oasc.Overseer.start Overseer (id=93583003170504707-127.0.0.1:25135__ww%2Fab-n_0000000000) starting
   [junit4]   2> 751643 T1332 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 751648 T1332 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 751649 T1360 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 751649 T1332 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 751650 T1359 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 751654 T1332 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/control-001/cores
   [junit4]   2> 751656 T1332 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=control_collection, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/control-001/cores/collection1/, name=collection1, config=solrconfig.xml, dataDir=data/, schema=schema.xml, loadOnStartup=true, coreNodeName=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/control-001/cores/collection1, shard=, transient=false}
   [junit4]   2> 751656 T1332 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/control-001/cores/collection1/
   [junit4]   2> 751656 T1332 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 751657 T1362 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 751658 T1362 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 751659 T1362 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 751659 T1358 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 751660 T1359 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25135/_ww/ab",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:25135__ww%2Fab",
   [junit4]   2> 	  "roles":null} current state version: 0
   [junit4]   2> 751660 T1359 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25135/_ww/ab",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:25135__ww%2Fab",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 751660 T1359 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 751661 T1359 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 751662 T1358 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 752660 T1362 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 752660 T1362 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 752661 T1362 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 752661 T1362 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 752662 T1362 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 752662 T1362 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/control-001/cores/collection1/'
   [junit4]   2> 752687 T1362 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 752690 T1362 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 752709 T1362 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 752762 T1362 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 752818 T1362 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 752819 T1362 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 752830 T1362 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 753286 T1362 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 753308 T1362 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 753312 T1362 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 753328 T1362 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 753334 T1362 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 753341 T1362 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 753343 T1362 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 753343 T1362 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 753344 T1362 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 753344 T1362 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 753344 T1362 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 753345 T1362 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 753345 T1362 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 753345 T1362 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/control-001/cores/collection1/data/
   [junit4]   2> 753345 T1362 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@29e4c4ed
   [junit4]   2> 753346 T1362 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/control-001/cores/collection1/data
   [junit4]   2> 753347 T1362 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/control-001/cores/collection1/data/index/
   [junit4]   2> 753347 T1362 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 753347 T1362 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/control-001/cores/collection1/data/index
   [junit4]   2> 753347 T1362 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=27, maxMergedSegmentMB=53.33203125, floorSegmentMB=1.701171875, forceMergeDeletesPctAllowed=13.492737200540025, segmentsPerTier=47.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1879079618145032
   [junit4]   2> 753348 T1362 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=RawDirectoryWrapper(RAMDirectory@4bd9b020 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@40ba2bc7),segFN=segments_1,generation=1}
   [junit4]   2> 753348 T1362 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 753354 T1362 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 753354 T1362 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 753355 T1362 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 753355 T1362 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 753355 T1362 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 753356 T1362 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 753356 T1362 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 753357 T1362 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 753357 T1362 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 753381 T1362 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 753386 T1362 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 753390 T1362 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 753393 T1362 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 753403 T1362 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/logging,/config,/get,/admin/properties,/admin/luke,/update/json,/admin/segments,/update,/admin/mbeans,/replication,/admin/file,/schema,/admin/ping,/update/json/docs,standard,/admin/plugins,/admin/system,/update/csv,/admin/threads
   [junit4]   2> 753406 T1362 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 753408 T1362 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 753408 T1362 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 753410 T1362 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 753410 T1362 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 753411 T1362 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=10, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 753412 T1362 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=RawDirectoryWrapper(RAMDirectory@4bd9b020 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@40ba2bc7),segFN=segments_1,generation=1}
   [junit4]   2> 753412 T1362 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 753412 T1362 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@38ae1e39[collection1] main
   [junit4]   2> 753413 T1362 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 753414 T1362 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 753414 T1362 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 753414 T1362 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 753415 T1362 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 753415 T1362 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 753415 T1362 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 753416 T1362 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 753416 T1362 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 753416 T1362 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 753419 T1363 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@38ae1e39[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 753420 T1362 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 753421 T1366 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:25135/_ww/ab collection:control_collection shard:shard1
   [junit4]   2> 753421 T1332 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 753421 T1332 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 753422 T1366 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 753423 T1332 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 753424 T1332 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 753427 T1370 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@d4e80da name:ZooKeeperConnection Watcher:127.0.0.1:25128/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 753427 T1332 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 753428 T1332 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 753428 T1332 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 753428 T1366 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 753432 T1358 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 753432 T1366 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 753433 T1366 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 753433 T1359 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C248 name=collection1 org.apache.solr.core.SolrCore@1aa95cd7 url=http://127.0.0.1:25135/_ww/ab/collection1 node=127.0.0.1:25135__ww%2Fab C248_STATE=coll:control_collection core:collection1 props:{core=collection1, state=down, base_url=http://127.0.0.1:25135/_ww/ab, node_name=127.0.0.1:25135__ww%2Fab}
   [junit4]   2> 753433 T1366 C:control_collection S:shard1 C248 P25135 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:25135/_ww/ab/collection1/
   [junit4]   2> 753433 T1332 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 753434 T1366 C:control_collection S:shard1 C248 P25135 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 753434 T1332 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 753435 T1366 C:control_collection S:shard1 C248 P25135 oasc.SyncStrategy.syncToMe http://127.0.0.1:25135/_ww/ab/collection1/ has no replicas
   [junit4]   2> 753435 T1358 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 753435 T1370 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 753435 T1332 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 753435 T1366 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:25135/_ww/ab/collection1/ shard1
   [junit4]   2> 753437 T1366 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 753437 T1332 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 753441 T1373 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1070bc26 name:ZooKeeperConnection Watcher:127.0.0.1:25128/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 753441 T1332 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 753441 T1332 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 753444 T1358 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 753446 T1359 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 2, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 753447 T1359 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 753450 T1359 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25135/_ww/ab",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 753451 T1359 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 753452 T1370 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 753452 T1358 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 753456 T1358 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 753457 T1370 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 753457 T1358 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 753498 T1366 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:25135/_ww/ab/collection1/ and leader is http://127.0.0.1:25135/_ww/ab/collection1/
   [junit4]   2> 753498 T1366 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:25135/_ww/ab
   [junit4]   2> 753499 T1366 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 753499 T1366 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 753499 T1366 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 753501 T1358 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 753502 T1359 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25135/_ww/ab",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:25135__ww%2Fab",
   [junit4]   2> 	  "roles":null} current state version: 4
   [junit4]   2> 753503 T1359 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25135/_ww/ab",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:25135__ww%2Fab",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 753607 T1358 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 753607 T1370 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 755088 T1332 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001/cores/collection1
   [junit4]   2> 755090 T1332 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001
   [junit4]   2> 755091 T1332 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 755094 T1332 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:25162
   [junit4]   2> 755095 T1332 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001/cores, hostPort=25161, solrconfig=solrconfig.xml, hostContext=/_ww/ab, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/tempDir-001/jetty1}
   [junit4]   2> 755095 T1332 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 755096 T1332 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001/'
   [junit4]   2> 755132 T1332 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001/solr.xml
   [junit4]   2> 755173 T1332 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001/cores
   [junit4]   2> 755174 T1332 oasc.CoreContainer.<init> New CoreContainer 2110856931
   [junit4]   2> 755174 T1332 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001/]
   [junit4]   2> 755174 T1332 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001/lib
   [junit4]   2> 755175 T1332 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001/lib).
   [junit4]   2> 755188 T1332 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 755188 T1332 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 755188 T1332 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 755189 T1332 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 755189 T1332 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 755189 T1332 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 755189 T1332 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 755190 T1332 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 755190 T1332 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 755190 T1332 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 755190 T1332 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 755191 T1332 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 755191 T1332 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 755191 T1332 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 755192 T1332 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 755192 T1332 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:25128/solr
   [junit4]   2> 755192 T1332 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 755192 T1332 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 755193 T1332 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 755195 T1385 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@21164eff name:ZooKeeperConnection Watcher:127.0.0.1:25128 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 755196 T1332 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 755197 T1332 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 755199 T1332 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 755200 T1388 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7daa8efb name:ZooKeeperConnection Watcher:127.0.0.1:25128/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 755200 T1332 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 755204 T1332 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 756208 T1332 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:25161__ww%2Fab
   [junit4]   2> 756209 T1332 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:25161__ww%2Fab
   [junit4]   2> 756212 T1332 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 756215 T1332 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001/cores
   [junit4]   2> 756217 T1332 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, schema=schema.xml, shard=, collection=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001/cores/collection1, config=solrconfig.xml, transient=false, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001/cores/collection1/, dataDir=data/, coreNodeName=, loadOnStartup=true}
   [junit4]   2> 756217 T1332 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001/cores/collection1/
   [junit4]   2> 756217 T1332 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 756219 T1389 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 756219 T1389 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 756221 T1358 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 756221 T1389 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 756221 T1389 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 756221 T1359 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25161/_ww/ab",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:25161__ww%2Fab",
   [junit4]   2> 	  "roles":null} current state version: 5
   [junit4]   2> 756222 T1389 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 756222 T1359 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25161/_ww/ab",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:25161__ww%2Fab",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 756222 T1359 oasco.ReplicaMutator.updateState Collection already exists with numShards=3
   [junit4]   2> 756222 T1359 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 756223 T1389 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 756223 T1389 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 756224 T1359 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 756224 T1388 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 756225 T1388 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 757224 T1389 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 757224 T1389 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 757225 T1389 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 757225 T1389 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 757225 T1389 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 757226 T1389 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001/cores/collection1/'
   [junit4]   2> 757249 T1389 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 757253 T1389 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 757265 T1389 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 757286 T1389 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 757330 T1389 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 757332 T1389 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 757341 T1389 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 757858 T1389 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 757880 T1389 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 757884 T1389 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 757899 T1389 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 757905 T1389 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 757911 T1389 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 757911 T1389 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 757912 T1389 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 757912 T1389 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 757913 T1389 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 757913 T1389 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 757913 T1389 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 757914 T1389 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 757914 T1389 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 757914 T1389 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@29e4c4ed
   [junit4]   2> 757915 T1389 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001/cores/collection1/data
   [junit4]   2> 757916 T1389 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 757916 T1389 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 757917 T1389 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 177722754B01BEA-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 757917 T1389 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=27, maxMergedSegmentMB=53.33203125, floorSegmentMB=1.701171875, forceMergeDeletesPctAllowed=13.492737200540025, segmentsPerTier=47.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1879079618145032
   [junit4]   2> 757918 T1389 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=RawDirectoryWrapper(RAMDirectory@3c50c7d0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5b6e479f),segFN=segments_1,generation=1}
   [junit4]   2> 757918 T1389 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 757927 T1389 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 757927 T1389 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 757928 T1389 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 757928 T1389 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 757928 T1389 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 757928 T1389 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 757929 T1389 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 757930 T1389 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 757930 T1389 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 757950 T1389 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 757954 T1389 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 757958 T1389 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 757961 T1389 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 757980 T1389 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/logging,/config,/get,/admin/properties,/admin/luke,/update/json,/admin/segments,/update,/admin/mbeans,/replication,/admin/file,/schema,/admin/ping,/update/json/docs,standard,/admin/plugins,/admin/system,/update/csv,/admin/threads
   [junit4]   2> 757982 T1389 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 757983 T1389 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 757983 T1389 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 757985 T1389 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 757985 T1389 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 757986 T1389 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=10, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 757987 T1389 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=RawDirectoryWrapper(RAMDirectory@3c50c7d0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5b6e479f),segFN=segments_1,generation=1}
   [junit4]   2> 757987 T1389 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 757987 T1389 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@34ba55b1[collection1] main
   [junit4]   2> 757988 T1389 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 757988 T1389 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 757988 T1389 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 757989 T1389 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 757989 T1389 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 757989 T1389 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 757990 T1389 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 757990 T1389 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 757990 T1389 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 757990 T1389 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 757992 T1390 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@34ba55b1[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 757992 T1389 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 757993 T1393 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:25161/_ww/ab collection:collection1 shard:shard1
   [junit4]   2> 757993 T1332 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 757994 T1332 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 757994 T1393 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 757997 T1393 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 757999 T1358 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 757999 T1393 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 757999 T1393 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C249 name=collection1 org.apache.solr.core.SolrCore@28eef1bd url=http://127.0.0.1:25161/_ww/ab/collection1 node=127.0.0.1:25161__ww%2Fab C249_STATE=coll:collection1 core:collection1 props:{core=collection1, state=down, base_url=http://127.0.0.1:25161/_ww/ab, node_name=127.0.0.1:25161__ww%2Fab}
   [junit4]   2> 757999 T1393 C:collection1 S:shard1 C249 P25161 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:25161/_ww/ab/collection1/
   [junit4]   2> 758000 T1359 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> ASYNC  NEW_CORE C250 name=collection1 org.apache.solr.core.SolrCore@28eef1bd url=http://127.0.0.1:25161/_ww/ab/collection1 node=127.0.0.1:25161__ww%2Fab C250_STATE=coll:collection1 core:collection1 props:{core=collection1, state=down, base_url=http://127.0.0.1:25161/_ww/ab, node_name=127.0.0.1:25161__ww%2Fab}
   [junit4]   2> 758011 T1393 C:collection1 S:shard1 C250 P25161 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 758011 T1393 C:collection1 S:shard1 C250 P25161 oasc.SyncStrategy.syncToMe http://127.0.0.1:25161/_ww/ab/collection1/ has no replicas
   [junit4]   2> 758012 T1393 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:25161/_ww/ab/collection1/ shard1
   [junit4]   2> 758012 T1393 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 758012 T1359 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 758013 T1388 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 758014 T1388 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 758015 T1359 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25161/_ww/ab",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 758016 T1359 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 758017 T1388 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 758018 T1388 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 758019 T1358 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 758066 T1393 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:25161/_ww/ab/collection1/ and leader is http://127.0.0.1:25161/_ww/ab/collection1/
   [junit4]   2> 758067 T1393 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:25161/_ww/ab
   [junit4]   2> 758067 T1393 C:collection1 S:shard1 oasc.ZkController.checkRecovery

[...truncated too long message...]

22776 T7433 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 1522779 T7322 C:control_collection S:shard1 c:collection1 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> 1522961 T7322 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:24981 24981
   [junit4]   2> 1523046 T7323 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:24981 24981
   [junit4]   2> 1523047 T7323 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 		2	/solr/collections/collection1/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1523047 T7322 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:25001/
   [junit4]   2> 1523047 T7322 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:25007/
   [junit4]   2> 1523047 T7322 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:24989/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=177722754B01BEA -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=sr_CS -Dtests.timezone=Pacific/Enderbury -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   70.0s J2 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([177722754B01BEA:89234DFDFA4C7612]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 1523069 T7322 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 177722754B01BEA-001
   [junit4]   2> 69981 T7321 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50), sim=DefaultSimilarity, locale=sr_CS, timezone=Pacific/Enderbury
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=182519552,total=326631424
   [junit4]   2> NOTE: All tests run in this JVM: [HighlighterMaxOffsetTest, TestRawResponseWriter, XsltUpdateRequestHandlerTest, TestFieldResource, TestStressReorder, TestRecoveryHdfs, AtomicUpdatesTest, OpenCloseCoreStressTest, TestReload, SchemaVersionSpecificBehaviorTest, DocValuesMultiTest, UniqFieldsUpdateProcessorFactoryTest, SpellPossibilityIteratorTest, SyncSliceTest, TestDistribDocBasedVersion, ShowFileRequestHandlerTest, TestRemoteStreaming, TestPivotHelperCode, InfoHandlerTest, TestTolerantSearch, SegmentsInfoRequestHandlerTest, SolrIndexConfigTest, TestReplicationHandler, DistribDocExpirationUpdateProcessorTest, TestArbitraryIndexDir, TestSolrDeletionPolicy2, SpellCheckComponentTest, TestFuzzyAnalyzedSuggestions, HdfsWriteToMultipleCollectionsTest, TestCollectionAPI, ZkNodePropsTest, TestSweetSpotSimilarityFactory, TestPerFieldSimilarity, TestFiltering, TestRequestStatusCollectionAPI, TestRecovery, TestManagedResource, EnumFieldTest, DebugComponentTest, TestDynamicFieldResource, HdfsDirectoryFactoryTest, TestClassNameShortening, BadCopyFieldTest, TestReplicaProperties, TestSolr4Spatial2, IndexSchemaTest, TestDFRSimilarityFactory, NumericFieldsTest, TestReRankQParserPlugin, ActionThrottleTest, SolrCoreTest, TestStressLucene, TestCodecSupport, TimeZoneUtilsTest, TestJsonFacets, CollectionsAPIDistributedZkTest, VMParamsZkACLAndCredentialsProvidersTest, CacheHeaderTest, TestDistributedSearch, BasicDistributedZkTest, TestSolrConfigHandlerCloud, TestConfigReload, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J2 in 70.76s, 1 test, 1 error <<< FAILURES!

[...truncated 700 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 55 minutes 44 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 236053084 bytes
Compression is 0.0%
Took 45 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2875 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2875/

4 tests failed.
REGRESSION:  org.apache.solr.search.facet.TestJsonFacets.testComplex

Error Message:
mismatch: 'civic'!='a' @ facets/makes/buckets/[0]/models/buckets/[1]/val

Stack Trace:
java.lang.RuntimeException: mismatch: 'civic'!='a' @ facets/makes/buckets/[0]/models/buckets/[1]/val
	at __randomizedtesting.SeedInfo.seed([7739CFC0B2BE137A:96E6CA5C9EF05F19]:0)
	at org.apache.solr.SolrTestCaseHS.matchJSON(SolrTestCaseHS.java:160)
	at org.apache.solr.SolrTestCaseHS.assertJQ(SolrTestCaseHS.java:142)
	at org.apache.solr.SolrTestCaseHS$Client.testJQ(SolrTestCaseHS.java:288)
	at org.apache.solr.search.facet.TestJsonFacets.testComplex(TestJsonFacets.java:155)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:51177/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:51177/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([7739CFC0B2BE137A:FF6DF01A1C427E82]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([7739CFC0B2BE137A:FF6DF01A1C427E82]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([7739CFC0B2BE137A:FF6DF01A1C427E82]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9329 lines...]
   [junit4] Suite: org.apache.solr.search.facet.TestJsonFacets
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.search.facet.TestJsonFacets 7739CFC0B2BE137A-001/init-core-data-001
   [junit4]   2> 229125 T547 oas.SolrTestCaseJ4.initCore ####initCore
   [junit4]   2> 229125 T547 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 229127 T547 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 229128 T547 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 229128 T547 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 229248 T547 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 229273 T547 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 229328 T547 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig-tlog.xml
   [junit4]   2> 229329 T547 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml
   [junit4]   2> 229337 T547 oass.IndexSchema.readSchema [null] Schema name=example
   [junit4]   2> 229575 T547 oass.AbstractSpatialFieldType.init WARN units parameter is deprecated, please use distanceUnits instead for field types with class SpatialRecursivePrefixTreeFieldType
   [junit4]   2> 229582 T547 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 229604 T547 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 229608 T547 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 229612 T547 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 229612 T547 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr
   [junit4]   2> 229612 T547 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/'
   [junit4]   2> 229651 T547 oasc.CoreContainer.<init> New CoreContainer 1879689010
   [junit4]   2> 229651 T547 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/]
   [junit4]   2> 229651 T547 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/lib
   [junit4]   2> 229652 T547 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/lib).
   [junit4]   2> 229662 T547 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 600000
   [junit4]   2> 229662 T547 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 229662 T547 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 60000
   [junit4]   2> 229662 T547 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 229663 T547 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 229663 T547 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 229663 T547 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 229663 T547 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 229663 T547 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 229664 T547 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 229664 T547 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 229664 T547 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=30000&connTimeout=30000&retry=true
   [junit4]   2> 229665 T547 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 229665 T547 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 229665 T547 oasc.CoreContainer.load Node Name: testNode
   [junit4]   2> 229667 T547 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {loadOnStartup=true, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.search.facet.TestJsonFacets 7739CFC0B2BE137A-001/init-core-data-001, collection=collection1, name=collection1, shard=shard1, instanceDir=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, transient=false, config=solrconfig-tlog.xml, schema=schema_latest.xml}
   [junit4]   2> 229667 T548 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 229669 T548 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 229670 T548 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 229670 T548 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 229757 T548 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 229782 T548 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 229823 T548 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig-tlog.xml
   [junit4]   2> 229824 T548 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml
   [junit4]   2> 229832 T548 oass.IndexSchema.readSchema [collection1] Schema name=example
   [junit4]   2> 230022 T548 oass.AbstractSpatialFieldType.init WARN units parameter is deprecated, please use distanceUnits instead for field types with class SpatialRecursivePrefixTreeFieldType
   [junit4]   2> 230030 T548 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 230046 T548 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 230051 T548 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 230056 T548 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 230056 T548 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 230056 T548 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.search.facet.TestJsonFacets 7739CFC0B2BE137A-001/init-core-data-001/
   [junit4]   2> 230057 T548 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5828ba35
   [junit4]   2> 230058 T548 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.search.facet.TestJsonFacets 7739CFC0B2BE137A-001/init-core-data-001
   [junit4]   2> 230058 T548 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.search.facet.TestJsonFacets 7739CFC0B2BE137A-001/init-core-data-001/index/
   [junit4]   2> 230058 T548 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.search.facet.TestJsonFacets 7739CFC0B2BE137A-001/init-core-data-001/index' doesn't exist. Creating new index...
   [junit4]   2> 230059 T548 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.search.facet.TestJsonFacets 7739CFC0B2BE137A-001/init-core-data-001/index
   [junit4]   2> 230059 T548 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=27, maxMergeAtOnceExplicit=18, maxMergedSegmentMB=16.2978515625, floorSegmentMB=0.353515625, forceMergeDeletesPctAllowed=16.295204050591646, segmentsPerTier=26.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 230060 T548 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6bb394b8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5ae8f345),segFN=segments_1,generation=1}
   [junit4]   2> 230061 T548 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 230068 T548 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 230068 T548 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 230069 T548 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 230069 T548 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 230070 T548 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 230070 T548 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 230070 T548 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 230071 T548 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 230071 T548 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 230101 T548 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 230104 T548 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 230107 T548 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 230109 T548 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 230116 T548 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/system,/admin/luke,/schema,standard,/admin/threads,/update/json,/admin/logging,/admin/properties,/admin/plugins,/update/json/docs,/admin/file,/get,/admin/mbeans,/update/csv,/admin/ping,/config,/update,/replication,/admin/segments
   [junit4]   2> 230118 T548 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 230119 T548 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 230120 T548 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 230121 T548 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 230121 T548 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 230122 T548 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=18, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=88.712890625, floorSegmentMB=1.6796875, forceMergeDeletesPctAllowed=20.970532846719955, segmentsPerTier=38.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.37418944491677153
   [junit4]   2> 230122 T548 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6bb394b8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5ae8f345),segFN=segments_1,generation=1}
   [junit4]   2> 230123 T548 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 230123 T548 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@51983697[collection1] main
   [junit4]   2> 230124 T548 c:collection1 oasr.ManagedResourceStorage.newStorageIO WARN Cannot write to config directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf; switching to use InMemory storage instead.
   [junit4]   2> 230125 T548 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 230125 T548 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 230125 T548 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 230125 T548 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 230126 T548 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 230127 T549 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@51983697[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 230129 T547 oas.SolrTestCaseJ4.initCore ####initCore end
   [junit4]   2> 230136 T547 oas.SolrTestCaseJ4.setUp ###Starting testStats
   [junit4]   2> ASYNC  NEW_CORE C84 name=collection1 org.apache.solr.core.SolrCore@37d51fe9
   [junit4]   2> 230140 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {deleteByQuery=*:* (-1497312353721516032)} 0 3
   [junit4]   2> 230147 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={versions=true&wt=json&indent=true} {add=[1 (1497312353729904640)]} 0 2
   [junit4]   2> 230149 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={versions=true&wt=json&indent=true} {add=[2 (1497312353733050368)]} 0 1
   [junit4]   2> 230150 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={versions=true&wt=json&indent=true} {add=[3 (1497312353735147520)]} 0 0
   [junit4]   2> 230151 T547 C84 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 230176 T547 C84 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6bb394b8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5ae8f345),segFN=segments_1,generation=1}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6bb394b8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5ae8f345),segFN=segments_2,generation=2}
   [junit4]   2> 230177 T547 C84 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
   [junit4]   2> 230186 T547 C84 oass.SolrIndexSearcher.<init> Opening Searcher@4b7fa047[collection1] main
   [junit4]   2> 230186 T547 C84 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 230187 T549 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4b7fa047[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):c3)))}
   [junit4]   2> 230188 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 38
   [junit4]   2> 230194 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={versions=true&wt=json&indent=true} {add=[4 (1497312353777090560)]} 0 4
   [junit4]   2> 230195 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={versions=true&wt=json&indent=true} {add=[5 (1497312353781284864)]} 0 1
   [junit4]   2> 230196 T547 C84 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 230210 T547 C84 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6bb394b8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5ae8f345),segFN=segments_2,generation=2}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6bb394b8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5ae8f345),segFN=segments_3,generation=3}
   [junit4]   2> 230211 T547 C84 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
   [junit4]   2> 230212 T547 C84 oass.SolrIndexSearcher.<init> Opening Searcher@18d90a23[collection1] main
   [junit4]   2> 230213 T547 C84 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 230214 T549 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@18d90a23[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):c3) Uninverting(_1(5.2.0):c2)))}
   [junit4]   2> 230215 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 19
   [junit4]   2> 230220 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={versions=true&wt=json&indent=true} {add=[6 (1497312353805402112)]} 0 3
   [junit4]   2> 230221 T547 C84 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 230228 T547 C84 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6bb394b8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5ae8f345),segFN=segments_3,generation=3}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6bb394b8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5ae8f345),segFN=segments_4,generation=4}
   [junit4]   2> 230228 T547 C84 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 4
   [junit4]   2> 230230 T547 C84 oass.SolrIndexSearcher.<init> Opening Searcher@57dc4a9d[collection1] main
   [junit4]   2> 230230 T547 C84 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 230231 T549 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@57dc4a9d[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.2.0):c3) Uninverting(_1(5.2.0):c2) Uninverting(_2(5.2.0):c1)))}
   [junit4]   2> 230232 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 12
   [junit4]   2> 230342 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7BcatA%3A%7Bquery%3A%7Bq%3A%27%24%7Bcat_s%7D%3AA%27%7D%7D%2C++catA2%3A%7Bquery%3A%7Bquery%3A%27%24%7Bcat_s%7D%3AA%27%7D%7D%2C++catA3%3A%7Bquery%3A%27%24%7Bcat_s%7D%3AA%27%7D++++%7D&wt=json&indent=true} hits=6 status=0 QTime=108 
   [junit4]   2> 230349 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7B+catB%3A%7Bquery%3A%7Bq%3A%27%24%7Bcat_s%7D%3AB%27%2C+facet%3A%7Bnj%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANJ%27%7D%2C+ny%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANY%27%7D%7D+%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=4 
   [junit4]   2> 230353 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=id%3A%282+3%29&json.facet=%7B+catB%3A%7Bquery%3A%7Bq%3A%27%24%7Bcat_s%7D%3AB%27%2C+facet%3A%7Bnj%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANJ%27%7D%2C+ny%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANY%27%7D%7D+%7D%7D%7D&wt=json&indent=true} hits=2 status=0 QTime=4 
   [junit4]   2> 230357 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7B+catB%3A%7Bquery%3A%7Bq%3A%27%24%7Bcat_s%7D%3AB%27%2C+facet%3A%7Bnj%3A%7Bquery%3A%7Bq%3A%27%24%7Bwhere_s%7D%3ANJ%27%7D%7D%2C+ny%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANY%27%7D%7D+%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=3 
   [junit4]   2> 230542 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bc1%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%7D%2C+c2%3A%7Bfield%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%7D%7D%2C+c3%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%7D%7D++%7D&wt=json&indent=true} hits=6 status=0 QTime=184 
   [junit4]   2> 230544 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+mincount%3A3%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 230546 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=id%3A1&json.facet=%7Bf1%3A%7Bterms%3A%27%24%7Bcat_s%7D%27%7D%7D&wt=json&indent=true} hits=1 status=0 QTime=2 
   [junit4]   2> 230547 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=id%3A1&json.facet=%7BprocessEmpty%3Atrue%2C+f1%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+mincount%3A0%7D%7D%7D&wt=json&indent=true} hits=1 status=0 QTime=1 
   [junit4]   2> 230702 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=id%3A1&json.facet=%7BprocessEmpty%3Atrue%2C+f1%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+mincount%3A0%2C+allBuckets%3Atrue%2C+facet%3A%7Bn1%3A%27sum%28%24%7Bnum_d%7D%29%27%7D++%7D%7D%7D&wt=json&indent=true} hits=1 status=0 QTime=154 
   [junit4]   2> 230705 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27n1+desc%27%2C+facet%3A%7Bn1%3A%27sum%28%24%7Bnum_d%7D%29%27%7D++%7D%7D+%2C+f2%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27n1+asc%27%2C+facet%3A%7Bn1%3A%27sum%28%24%7Bnum_d%7D%29%27%7D++%7D%7D+%7D&wt=json&indent=true} hits=6 status=0 QTime=2 
   [junit4]   2> 230793 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27n1+desc%27%2C+facet%3A%7Bn1%3A%27percentile%28%24%7Bnum_d%7D%2C50%29%27%7D++%7D%7D+%2C+f2%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27n1+asc%27%2C+facet%3A%7Bn1%3A%27percentile%28%24%7Bnum_d%7D%2C50%29%27%7D++%7D%7D+%7D&wt=json&indent=true} hits=6 status=0 QTime=88 
   [junit4]   2> 230796 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27n1+desc%27%2C+facet%3A%7Bn1%3A%27percentile%28%24%7Bnum_d%7D%2C50%2C0%2C100%29%27%7D++%7D%7D+%2C+f2%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27n1+asc%27%2C+facet%3A%7Bn1%3A%27percentile%28%24%7Bnum_d%7D%2C50%2C0%2C100%29%27%7D++%7D%7D+%7D&wt=json&indent=true} hits=6 status=0 QTime=2 
   [junit4]   2> 230799 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27count+desc%27+%7D++%7D+++++++++++%2C+f2%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27count+asc%27++%7D++%7D+++++++++++%2C+f3%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27index+asc%27++%7D++%7D+++++++++++%2C+f4%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27index+desc%27+%7D++%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=2 
   [junit4]   2> 230802 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=id%3A1+id%3A6&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27count+desc%27+%7D++%7D+++++++++++%2C+f2%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27count+asc%27++%7D++%7D%7D&wt=json&indent=true} hits=2 status=0 QTime=2 
   [junit4]   2> 230806 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bcat%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+facet%3A%7Bnj%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANJ%27%7D%7D++++%7D+++%7D%7D+%7D&wt=json&indent=true} hits=6 status=0 QTime=3 
   [junit4]   2> 230811 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=id%3A%282+5+4%29&json.facet=%7Bcat%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+facet%3A%7Bnj%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANJ%27%7D%7D++++%7D+++%7D%7D+%7D&wt=json&indent=true} hits=3 status=0 QTime=4 
   [junit4]   2> 230814 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bsuper_s%7D%2C+prefix%3As%2C+mincount%3A0+%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=2 
   [junit4]   2> 230815 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bsuper_s%7D%2C+prefix%3Attt%2C+mincount%3A0+%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 230816 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bsuper_s%7D%2C+prefix%3Aaaaaaa%2C+mincount%3A0+%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=0 
   [junit4]   2> 230818 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bsuper_s%7D%2C+prefix%3Azzzzzz%2C+mincount%3A0+%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 230819 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bnoexist%7D%2C+missing%3Atrue%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 230820 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bsparse_s%7D%2C+missing%3Atrue+%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 230822 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bsparse_s%7D%2C+missing%3Atrue%2C+facet%3A%7Bx%3A%27sum%28num_d%29%27%7D+++%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 230823 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bsparse_s%7D%2C+missing%3Atrue%2C+prefix%3Aon%2C+facet%3A%7Bx%3A%27sum%28num_d%29%27%7D+++%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=0 
   [junit4]   2> 230836 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bsparse_s%7D%2C+missing%3Atrue%2C+prefix%3Appp%2C+facet%3A%7Bx%3A%27sum%28num_d%29%27%7D+++%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=12 
   [junit4]   2> 230838 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&facet=true&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bcat_s%7D%2C+numBuckets%3Atrue%2C+limit%3A1%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 230839 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&facet=true&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bcat_s%7D%2C+numBuckets%3Atrue%2C+prefix%3AB%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=0 
   [junit4]   2> 230841 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&facet=true&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bcat_s%7D%2C+numBuckets%3Atrue%2C+mincount%3A3%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 230944 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf%3A%7Brange%3A%7Bfield%3A%24%7Bnum_d%7D%2C+start%3A-5%2C+end%3A10%2C+gap%3A5%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=102 
   [junit4]   2> 230948 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf%3A%7Brange%3A%7Bfield%3A%24%7Bnum_d%7D%2C+start%3A-5%2C+end%3A10%2C+gap%3A5%2C+include%3Aupper%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=2 
   [junit4]   2> 231029 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf%3A%7Brange%3A%7Bfield%3A%24%7Bnum_d%7D%2C+start%3A-5%2C+end%3A10%2C+gap%3A5%2C+++facet%3A%7B+x%3A%27sum%28%24%7Bnum_i%7D%29%27%2C+ny%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANY%27%7D%7D+++%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=80 
   [junit4]   2> 231038 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf%3A%7Brange%3A%7Bfield%3A%24%7Bnum_d%7D%2C+start%3A-5%2C+end%3A10%2C+gap%3A5%2C+other%3Aall%2C+++facet%3A%7B+x%3A%27sum%28%24%7Bnum_i%7D%29%27%2C+ny%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANY%27%7D%7D+++%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=7 
   [junit4]   2> 231046 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=id%3A%283+4+6%29&json.facet=%7Bf%3A%7Brange%3A%7Bfield%3A%24%7Bnum_d%7D%2C+start%3A-5%2C+end%3A10%2C+gap%3A5%2C+other%3Aall%2C+++facet%3A%7B+x%3A%27sum%28%24%7Bnum_i%7D%29%27%2C+ny%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANY%27%7D%7D+++%7D%7D%7D&wt=json&indent=true} hits=3 status=0 QTime=7 
   [junit4]   2> 231164 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7B+sum1%3A%27sum%28%24%7Bnum_d%7D%29%27%2C+sumsq1%3A%27sumsq%28%24%7Bnum_d%7D%29%27%2C+avg1%3A%27avg%28%24%7Bnum_d%7D%29%27%2C+min1%3A%27min%28%24%7Bnum_d%7D%29%27%2C+max1%3A%27max%28%24%7Bnum_d%7D%29%27%2C+numwhere%3A%27unique%28%24%7Bwhere_s%7D%29%27%2C+med%3A%27percentile%28%24%7Bnum_d%7D%2C50%29%27%2C+perc%3A%27percentile%28%24%7Bnum_d%7D%2C0%2C50.0%2C100%29%27+%7D&wt=json&indent=true} hits=6 status=0 QTime=117 
   [junit4]   2> 231167 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=id%3ADOESNOTEXIST&json.facet=%7B+sum1%3A%27sum%28%24%7Bnum_d%7D%29%27%2C+sumsq1%3A%27sumsq%28%24%7Bnum_d%7D%29%27%2C+avg1%3A%27avg%28%24%7Bnum_d%7D%29%27%2C+min1%3A%27min%28%24%7Bnum_d%7D%29%27%2C+max1%3A%27max%28%24%7Bnum_d%7D%29%27%2C+numwhere%3A%27unique%28%24%7Bwhere_s%7D%29%27%2C+med%3A%27percentile%28%24%7Bnum_d%7D%2C50%29%27%2C+perc%3A%27percentile%28%24%7Bnum_d%7D%2C0%2C50.0%2C100%29%27+%7D&wt=json&indent=true} hits=0 status=0 QTime=2 
   [junit4]   2> 231221 T547 C84 oassf.UnInvertedField.<init> UnInverted multi-valued field {field=multi_ss,memSize=4515,tindexSize=41,time=20,phase1=20,nTerms=2,bigTerms=2,termInstances=0,uses=0}
   [junit4]   2> 231226 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bcat%3A%7Bterms%3A%7Bfield%3A%27%24%7Bmulti_ss%7D%27%2C+facet%3A%7Bnj%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANJ%27%7D%7D++++%7D+++%7D%7D+%7D&wt=json&indent=true} hits=6 status=0 QTime=59 
   [junit4]   2> 231240 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bx%3A%27unique%28%24%7Bmulti_ss%7D%29%27%2C+y%3A%7Bquery%3A%7Bq%3A%27id%3A2%27%2C+facet%3A%7Bx%3A%27unique%28%24%7Bmulti_ss%7D%29%27%7D+%7D%7D+++%7D&wt=json&indent=true} hits=6 status=0 QTime=14 
   [junit4]   2> 231243 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bx%3A%7Bterms%3A%7Bfield%3A%27%24%7Bmulti_ss%7D%27%2CallBuckets%3Atrue%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=2 
   [junit4]   2> 231255 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&facet=true&facet.version=2&facet.field=%7B%21key%3Df1%7D%24%7Bcat_s%7D&facet.mincount=3&wt=json&indent=true} hits=6 status=0 QTime=12 
   [junit4]   2> 231257 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&facet=true&facet.version=2&facet.field=%7B%21key%3Df1%7D%24%7Bsuper_s%7D&facet.prefix=s&facet.mincount=0&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 231273 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&facet=true&facet.version=2&facet.range=%7B%21key%3Df%7D%24%7Bnum_d%7D&facet.range.start=-5&facet.range.end=10&facet.range.gap=5&f.f.facet.stat=x%3Asum%28%24%7Bnum_i%7D%29&subfacet.f.query=%7B%21key%3Dny%7D%24%7Bwhere_s%7D%3ANY&wt=json&indent=true} hits=6 status=0 QTime=15 
   [junit4]   2> 231276 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=*%3A*&facet=true&facet.version=2&facet.field=%7B%21key%3Df1%7D%24%7Bcat_s%7D&facet.field=%7B%21key%3Df2%7D%24%7Bcat_s%7D&f.f1.facet.sort=n1+desc&f.f1.facet.sort=n1+asc&facet.stat=n1%3Asum%28num_d%29&wt=json&indent=true} hits=6 status=0 QTime=2 
   [junit4]   2> 231287 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={rows=0&noexist=noexist_s&cat_s=cat_s&where_s=where_s&num_d=num_d&num_i=num_i&super_s=super_s&val_b=val_b&sparse_s=sparse_s&multi_ss=multi_ss&q=id%3A%283+4+6%29&facet=true&facet.version=2&facet.range=%7B%21key%3Df%7D%24%7Bnum_d%7D&facet.range.start=-5&facet.range.end=10&facet.range.gap=5&f.f.facet.stat=x%3Asum%28%24%7Bnum_i%7D%29&subfacet.f.query=%7B%21key%3Dny%7D%24%7Bwhere_s%7D%3ANY&facet.range.other=all&wt=json&indent=true} hits=3 status=0 QTime=10 
   [junit4]   2> 231293 T547 C84 oass.SolrIndexSearcher.<init> Opening Searcher@6c63f619[collection1] realtime
   [junit4]   2> 231293 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {deleteByQuery=*:* (-1497312354928427008)} 0 5
   [junit4]   2> 231303 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={versions=true&wt=json&indent=true} {add=[1 (1497312354942058496)]} 0 2
   [junit4]   2> 231304 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={versions=true&wt=json&indent=true} {add=[2 (1497312354944155648)]} 0 1
   [junit4]   2> 231305 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={versions=true&wt=json&indent=true} {add=[3 (1497312354946252800)]} 0 1
   [junit4]   2> 231305 T547 C84 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 231333 T547 C84 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6bb394b8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5ae8f345),segFN=segments_4,generation=4}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6bb394b8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5ae8f345),segFN=segments_5,generation=5}
   [junit4]   2> 231334 T547 C84 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 5
   [junit4]   2> 231349 T547 C84 oass.SolrIndexSearcher.<init> Opening Searcher@7313cfa2[collection1] main
   [junit4]   2> 231350 T547 C84 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 231351 T549 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7313cfa2[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_3(5.2.0):c3)))}
   [junit4]   2> 231352 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 47
   [junit4]   2> 231358 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={versions=true&wt=json&indent=true} {add=[4 (1497312354997633024)]} 0 4
   [junit4]   2> 231359 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={versions=true&wt=json&indent=true} {add=[5 (1497312355001827328)]} 0 1
   [junit4]   2> 231360 T547 C84 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 231368 T547 C84 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6bb394b8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5ae8f345),segFN=segments_5,generation=5}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6bb394b8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5ae8f345),segFN=segments_6,generation=6}
   [junit4]   2> 231368 T547 C84 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 6
   [junit4]   2> 231370 T547 C84 oass.SolrIndexSearcher.<init> Opening Searcher@4cb88286[collection1] main
   [junit4]   2> 231370 T547 C84 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 231371 T549 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4cb88286[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_3(5.2.0):c3) Uninverting(_4(5.2.0):c2)))}
   [junit4]   2> 231372 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 13
   [junit4]   2> 231377 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={versions=true&wt=json&indent=true} {add=[6 (1497312355018604544)]} 0 3
   [junit4]   2> 231378 T547 C84 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 231387 T547 C84 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6bb394b8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5ae8f345),segFN=segments_6,generation=6}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6bb394b8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5ae8f345),segFN=segments_7,generation=7}
   [junit4]   2> 231388 T547 C84 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 7
   [junit4]   2> 231390 T547 C84 oass.SolrIndexSearcher.<init> Opening Searcher@2bd27391[collection1] main
   [junit4]   2> 231391 T547 C84 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 231391 T549 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2bd27391[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_3(5.2.0):c3) Uninverting(_4(5.2.0):c2) Uninverting(_5(5.2.0):c1)))}
   [junit4]   2> 231392 T547 C84 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 14
   [junit4]   2> 231414 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7BcatA%3A%7Bquery%3A%7Bq%3A%27%24%7Bcat_s%7D%3AA%27%7D%7D%2C++catA2%3A%7Bquery%3A%7Bquery%3A%27%24%7Bcat_s%7D%3AA%27%7D%7D%2C++catA3%3A%7Bquery%3A%27%24%7Bcat_s%7D%3AA%27%7D++++%7D&wt=json&indent=true} hits=6 status=0 QTime=20 
   [junit4]   2> 231417 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7B+catB%3A%7Bquery%3A%7Bq%3A%27%24%7Bcat_s%7D%3AB%27%2C+facet%3A%7Bnj%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANJ%27%7D%2C+ny%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANY%27%7D%7D+%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=3 
   [junit4]   2> 231422 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=id%3A%282+3%29&json.facet=%7B+catB%3A%7Bquery%3A%7Bq%3A%27%24%7Bcat_s%7D%3AB%27%2C+facet%3A%7Bnj%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANJ%27%7D%2C+ny%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANY%27%7D%7D+%7D%7D%7D&wt=json&indent=true} hits=2 status=0 QTime=4 
   [junit4]   2> 231425 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7B+catB%3A%7Bquery%3A%7Bq%3A%27%24%7Bcat_s%7D%3AB%27%2C+facet%3A%7Bnj%3A%7Bquery%3A%7Bq%3A%27%24%7Bwhere_s%7D%3ANJ%27%7D%7D%2C+ny%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANY%27%7D%7D+%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=2 
   [junit4]   2> 231427 T547 C84 oassf.UnInvertedField.<init> UnInverted multi-valued field {field=cat_ss,memSize=4446,tindexSize=41,time=0,phase1=0,nTerms=2,bigTerms=1,termInstances=2,uses=0}
   [junit4]   2> 231430 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bc1%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%7D%2C+c2%3A%7Bfield%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%7D%7D%2C+c3%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%7D%7D++%7D&wt=json&indent=true} hits=6 status=0 QTime=4 
   [junit4]   2> 231432 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+mincount%3A3%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 231434 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=id%3A1&json.facet=%7Bf1%3A%7Bterms%3A%27%24%7Bcat_s%7D%27%7D%7D&wt=json&indent=true} hits=1 status=0 QTime=1 
   [junit4]   2> 231437 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=id%3A1&json.facet=%7BprocessEmpty%3Atrue%2C+f1%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+mincount%3A0%7D%7D%7D&wt=json&indent=true} hits=1 status=0 QTime=2 
   [junit4]   2> 231448 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=id%3A1&json.facet=%7BprocessEmpty%3Atrue%2C+f1%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+mincount%3A0%2C+allBuckets%3Atrue%2C+facet%3A%7Bn1%3A%27sum%28%24%7Bnum_d%7D%29%27%7D++%7D%7D%7D&wt=json&indent=true} hits=1 status=0 QTime=11 
   [junit4]   2> 231451 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27n1+desc%27%2C+facet%3A%7Bn1%3A%27sum%28%24%7Bnum_d%7D%29%27%7D++%7D%7D+%2C+f2%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27n1+asc%27%2C+facet%3A%7Bn1%3A%27sum%28%24%7Bnum_d%7D%29%27%7D++%7D%7D+%7D&wt=json&indent=true} hits=6 status=0 QTime=2 
   [junit4]   2> 231454 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27n1+desc%27%2C+facet%3A%7Bn1%3A%27percentile%28%24%7Bnum_d%7D%2C50%29%27%7D++%7D%7D+%2C+f2%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27n1+asc%27%2C+facet%3A%7Bn1%3A%27percentile%28%24%7Bnum_d%7D%2C50%29%27%7D++%7D%7D+%7D&wt=json&indent=true} hits=6 status=0 QTime=2 
   [junit4]   2> 231457 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27n1+desc%27%2C+facet%3A%7Bn1%3A%27percentile%28%24%7Bnum_d%7D%2C50%2C0%2C100%29%27%7D++%7D%7D+%2C+f2%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27n1+asc%27%2C+facet%3A%7Bn1%3A%27percentile%28%24%7Bnum_d%7D%2C50%2C0%2C100%29%27%7D++%7D%7D+%7D&wt=json&indent=true} hits=6 status=0 QTime=2 
   [junit4]   2> 231463 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27count+desc%27+%7D++%7D+++++++++++%2C+f2%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27count+asc%27++%7D++%7D+++++++++++%2C+f3%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27index+asc%27++%7D++%7D+++++++++++%2C+f4%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27index+desc%27+%7D++%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=5 
   [junit4]   2> 231467 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=id%3A1+id%3A6&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27count+desc%27+%7D++%7D+++++++++++%2C+f2%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+sort%3A%27count+asc%27++%7D++%7D%7D&wt=json&indent=true} hits=2 status=0 QTime=3 
   [junit4]   2> 231471 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bcat%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+facet%3A%7Bnj%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANJ%27%7D%7D++++%7D+++%7D%7D+%7D&wt=json&indent=true} hits=6 status=0 QTime=3 
   [junit4]   2> 231477 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=id%3A%282+5+4%29&json.facet=%7Bcat%3A%7Bterms%3A%7Bfield%3A%27%24%7Bcat_s%7D%27%2C+facet%3A%7Bnj%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANJ%27%7D%7D++++%7D+++%7D%7D+%7D&wt=json&indent=true} hits=3 status=0 QTime=5 
   [junit4]   2> 231479 T547 C84 oassf.UnInvertedField.<init> UnInverted multi-valued field {field=super_ss,memSize=4358,tindexSize=46,time=1,phase1=1,nTerms=5,bigTerms=0,termInstances=5,uses=0}
   [junit4]   2> 231479 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bsuper_s%7D%2C+prefix%3As%2C+mincount%3A0+%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 231481 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bsuper_s%7D%2C+prefix%3Attt%2C+mincount%3A0+%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 231482 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bsuper_s%7D%2C+prefix%3Aaaaaaa%2C+mincount%3A0+%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 231483 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bsuper_s%7D%2C+prefix%3Azzzzzz%2C+mincount%3A0+%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=0 
   [junit4]   2> 231484 T547 C84 oassf.UnInvertedField.<init> UnInverted multi-valued field {field=noexist_ss,memSize=4288,tindexSize=0,time=0,phase1=0,nTerms=0,bigTerms=0,termInstances=0,uses=0}
   [junit4]   2> 231485 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bnoexist%7D%2C+missing%3Atrue%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 231487 T547 C84 oassf.UnInvertedField.<init> UnInverted multi-valued field {field=sparse_ss,memSize=4355,tindexSize=43,time=1,phase1=1,nTerms=2,bigTerms=0,termInstances=2,uses=0}
   [junit4]   2> 231487 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bsparse_s%7D%2C+missing%3Atrue+%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 231489 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bsparse_s%7D%2C+missing%3Atrue%2C+facet%3A%7Bx%3A%27sum%28num_d%29%27%7D+++%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 231491 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bsparse_s%7D%2C+missing%3Atrue%2C+prefix%3Aon%2C+facet%3A%7Bx%3A%27sum%28num_d%29%27%7D+++%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 231493 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bsparse_s%7D%2C+missing%3Atrue%2C+prefix%3Appp%2C+facet%3A%7Bx%3A%27sum%28num_d%29%27%7D+++%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 231495 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&facet=true&rows=0&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bcat_s%7D%2C+numBuckets%3Atrue%2C+limit%3A1%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=2 
   [junit4]   2> 231496 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&facet=true&rows=0&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bcat_s%7D%2C+numBuckets%3Atrue%2C+prefix%3AB%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 231498 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&facet=true&rows=0&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf1%3A%7Bterms%3A%7Bfield%3A%24%7Bcat_s%7D%2C+numBuckets%3Atrue%2C+mincount%3A3%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=1 
   [junit4]   2> 231501 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf%3A%7Brange%3A%7Bfield%3A%24%7Bnum_d%7D%2C+start%3A-5%2C+end%3A10%2C+gap%3A5%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=2 
   [junit4]   2> 231504 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf%3A%7Brange%3A%7Bfield%3A%24%7Bnum_d%7D%2C+start%3A-5%2C+end%3A10%2C+gap%3A5%2C+include%3Aupper%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=2 
   [junit4]   2> 231508 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf%3A%7Brange%3A%7Bfield%3A%24%7Bnum_d%7D%2C+start%3A-5%2C+end%3A10%2C+gap%3A5%2C+++facet%3A%7B+x%3A%27sum%28%24%7Bnum_i%7D%29%27%2C+ny%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANY%27%7D%7D+++%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=3 
   [junit4]   2> 231515 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7Bf%3A%7Brange%3A%7Bfield%3A%24%7Bnum_d%7D%2C+start%3A-5%2C+end%3A10%2C+gap%3A5%2C+other%3Aall%2C+++facet%3A%7B+x%3A%27sum%28%24%7Bnum_i%7D%29%27%2C+ny%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANY%27%7D%7D+++%7D%7D%7D&wt=json&indent=true} hits=6 status=0 QTime=6 
   [junit4]   2> 231522 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=id%3A%283+4+6%29&json.facet=%7Bf%3A%7Brange%3A%7Bfield%3A%24%7Bnum_d%7D%2C+start%3A-5%2C+end%3A10%2C+gap%3A5%2C+other%3Aall%2C+++facet%3A%7B+x%3A%27sum%28%24%7Bnum_i%7D%29%27%2C+ny%3A%7Bquery%3A%27%24%7Bwhere_s%7D%3ANY%27%7D%7D+++%7D%7D%7D&wt=json&indent=true} hits=3 status=0 QTime=6 
   [junit4]   2> 231524 T547 C84 oassf.UnInvertedField.<init> UnInverted multi-valued field {field=where_ss,memSize=4448,tindexSize=42,time=0,phase1=0,nTerms=2,bigTerms=1,termInstances=2,uses=0}
   [junit4]   2> 231526 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss&q=*%3A*&json.facet=%7B+sum1%3A%27sum%28%24%7Bnum_d%7D%29%27%2C+sumsq1%3A%27sumsq%28%24%7Bnum_d%7D%29%27%2C+avg1%3A%27avg%28%24%7Bnum_d%7D%29%27%2C+min1%3A%27min%28%24%7Bnum_d%7D%29%27%2C+max1%3A%27max%28%24%7Bnum_d%7D%29%27%2C+numwhere%3A%27unique%28%24%7Bwhere_s%7D%29%27%2C+med%3A%27percentile%28%24%7Bnum_d%7D%2C50%29%27%2C+perc%3A%27percentile%28%24%7Bnum_d%7D%2C0%2C50.0%2C100%29%27+%7D&wt=json&indent=true} hits=6 status=0 QTime=3 
   [junit4]   2> 231528 T547 C84 oasc.SolrCore.execute [collection1] webapp=null path=null params={facet=true&rows=0&noexist=noexist_ss&cat_s=cat_ss&where_s=where_ss&num_d=num_d&num_i=num_i&super_s=super_ss&val_b=val_b&sparse_s=sparse_ss&multi_ss=multi_ss

[...truncated too long message...]


   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 		4	/solr/collections/collection1/state.json
   [junit4]   2> 		3	/solr/collections/c8n_1x3_commits/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent data watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		2	/solr/collections/c8n_1x3_commits/leader_elect/shard1/election/93582112433373197-core_node3-n_0000000000
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 		4	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1689972 T5544 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:51201/
   [junit4]   2> 1689972 T5544 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:51161/
   [junit4]   2> 1689972 T5544 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 3 connections to: http://127.0.0.1:51185/
   [junit4]   2> 1689973 T5544 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:51215/
   [junit4]   2> 1689973 T5544 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 13 connections to: http://127.0.0.1:51177/
   [junit4]   2> 1689973 T5544 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:51201/
   [junit4]   2> 1689974 T5544 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:51161/
   [junit4]   2> 1689974 T5544 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:51185/
   [junit4]   2> 1689974 T5544 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:51215/
   [junit4]   2> 1689975 T5544 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:51177/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderInitiatedRecoveryOnCommitTest -Dtests.method=test -Dtests.seed=7739CFC0B2BE137A -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ja_JP_JP_#u-ca-japanese -Dtests.timezone=Asia/Phnom_Penh -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   81.4s J2 | LeaderInitiatedRecoveryOnCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:51177/c8n_1x3_commits_shard1_replica2
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([7739CFC0B2BE137A:FF6DF01A1C427E82]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 48 more
   [junit4]   2> 1689999 T5544 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7739CFC0B2BE137A-001
   [junit4]   2> 81357 T5543 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=DefaultSimilarity, locale=ja_JP_JP_#u-ca-japanese, timezone=Asia/Phnom_Penh
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=100946744,total=374865920
   [junit4]   2> NOTE: All tests run in this JVM: [LeaderElectionIntegrationTest, ParsingFieldUpdateProcessorsTest, DistributedFacetPivotLongTailTest, CachingDirectoryFactoryTest, TestTrie, TestMergePolicyConfig, TestMacros, TestDistribDocBasedVersion, TestFieldTypeCollectionResource, DistributedFacetPivotSmallTest, TestStressVersions, TestXIncludeConfig, DateFieldTest, DebugComponentTest, SpellingQueryConverterTest, AddSchemaFieldsUpdateProcessorFactoryTest, TestNoOpRegenerator, TestObjectReleaseTracker, CSVRequestHandlerTest, OutputWriterTest, TestConfigOverlay, TestReload, TestLRUStatsCache, TermsComponentTest, ReplicationFactorTest, CustomCollectionTest, TestIndexingPerformance, TestSolrIndexConfig, SpatialRPTFieldTypeTest, BasicDistributedZkTest, BadIndexSchemaTest, TestManagedResourceStorage, TestSearchPerf, TestCharFilters, TestLRUCache, SolrInfoMBeanTest, TestSolrDeletionPolicy1, TestDocBasedVersionConstraints, DeleteLastCustomShardedReplicaTest, LukeRequestHandlerTest, TestHashPartitioner, RecoveryZkTest, BadCopyFieldTest, QueryEqualityTest, TestRangeQuery, TestQuerySenderNoQuery, SchemaVersionSpecificBehaviorTest, TestHdfsUpdateLog, HighlighterMaxOffsetTest, TestDefaultStatsCache, TestDFRSimilarityFactory, RegexBoostProcessorTest, CollectionsAPIDistributedZkTest, DateMathParserTest, OverriddenZkACLAndCredentialsProvidersTest, TestFiltering, TestSimpleQParserPlugin, HighlighterConfigTest, HdfsUnloadDistributedZkTest, TestStressUserVersions, SyncSliceTest, TestCloudManagedSchemaConcurrent, UnloadDistributedZkTest, LeaderInitiatedRecoveryOnCommitTest]
   [junit4] Completed on J2 in 81.64s, 1 test, 1 error <<< FAILURES!

[...truncated 765 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 4 errors, 61 ignored (21 assumptions)

Total time: 57 minutes 1 second
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 218418265 bytes
Compression is 0.0%
Took 44 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2874 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2874/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:48476/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48476/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([E7E5C63DFB521D73:6FB1F9E755AE708B]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48337/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48337/collection1
	at __randomizedtesting.SeedInfo.seed([E7E5C63DFB521D73:6FB1F9E755AE708B]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48337/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28719/jx/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28719/jx/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([E7E5C63DFB521D73:6FB1F9E755AE708B]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28719/jx/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9372 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/init-core-data-001
   [junit4]   2> 210845 T861 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /jx/
   [junit4]   2> 210850 T861 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 210851 T862 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 210951 T861 oasc.ZkTestServer.run start zk server on port:25294
   [junit4]   2> 210952 T861 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 210953 T861 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 210957 T869 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@233730e0 name:ZooKeeperConnection Watcher:127.0.0.1:25294 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 210957 T861 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 210957 T861 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 210958 T861 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 210960 T861 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 210961 T861 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 210962 T872 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@69a681f4 name:ZooKeeperConnection Watcher:127.0.0.1:25294/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 210962 T861 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 210963 T861 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 210963 T861 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 210965 T861 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 210966 T861 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 210967 T861 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 210968 T861 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 210969 T861 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 210971 T861 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 210971 T861 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 210973 T861 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 210973 T861 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 210975 T861 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 210975 T861 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 210977 T861 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 210977 T861 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 210979 T861 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 210979 T861 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 210981 T861 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 210981 T861 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 210983 T861 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 210983 T861 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 210985 T861 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 210985 T861 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 210987 T861 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 210987 T861 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 210989 T861 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 210989 T861 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 212149 T861 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/control-001/cores/collection1
   [junit4]   2> 212153 T861 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 212157 T861 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:25300
   [junit4]   2> 212157 T861 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/jx, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/control-001/cores, hostPort=25299, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/tempDir-001/control/data}
   [junit4]   2> 212158 T861 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@5fb271f1
   [junit4]   2> 212158 T861 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/control-001/'
   [junit4]   2> 212199 T861 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/control-001/solr.xml
   [junit4]   2> 212250 T861 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/control-001/cores
   [junit4]   2> 212251 T861 oasc.CoreContainer.<init> New CoreContainer 1996701156
   [junit4]   2> 212251 T861 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/control-001/]
   [junit4]   2> 212251 T861 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/control-001/lib
   [junit4]   2> 212252 T861 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/control-001/lib).
   [junit4]   2> 212263 T861 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 212263 T861 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 212263 T861 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 212263 T861 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 212264 T861 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 212264 T861 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 212264 T861 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 212265 T861 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 212265 T861 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 212265 T861 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 212265 T861 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 212266 T861 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 212266 T861 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 212266 T861 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 212267 T861 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 212267 T861 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:25294/solr
   [junit4]   2> 212268 T861 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 212268 T861 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 212269 T861 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 212270 T884 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7204b839 name:ZooKeeperConnection Watcher:127.0.0.1:25294 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 212270 T861 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 212271 T861 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 212273 T861 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 212274 T887 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4fed9c09 name:ZooKeeperConnection Watcher:127.0.0.1:25294/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 212274 T861 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 212276 T861 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 212277 T861 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 212279 T861 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 212280 T861 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 212281 T861 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 212283 T861 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 212284 T861 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 212285 T861 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 212286 T861 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:25299_jx
   [junit4]   2> 212286 T861 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:25299_jx
   [junit4]   2> 212287 T861 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 212288 T861 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 212289 T861 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 212291 T861 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:25299_jx
   [junit4]   2> 212291 T861 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 212292 T861 oasc.Overseer.start Overseer (id=93581231638642691-127.0.0.1:25299_jx-n_0000000000) starting
   [junit4]   2> 212293 T861 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 212297 T861 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 212298 T889 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 212298 T861 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 212298 T888 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 212301 T861 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/control-001/cores
   [junit4]   2> 212303 T861 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {transient=false, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/control-001/cores/collection1, dataDir=data/, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/control-001/cores/collection1/, config=solrconfig.xml, loadOnStartup=true, shard=, name=collection1, collection=control_collection, coreNodeName=}
   [junit4]   2> 212303 T861 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/control-001/cores/collection1/
   [junit4]   2> 212304 T861 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 212305 T891 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 212305 T891 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 212306 T891 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 212306 T887 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 212307 T888 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:25299_jx",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25299/jx"} current state version: 0
   [junit4]   2> 212307 T888 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:25299_jx",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25299/jx"}
   [junit4]   2> 212307 T888 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 212308 T888 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 212309 T887 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 213306 T891 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 213307 T891 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 213308 T891 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 213308 T891 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 213308 T891 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 213309 T891 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/control-001/cores/collection1/'
   [junit4]   2> 213323 T891 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 213325 T891 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 213337 T891 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 213357 T891 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 213411 T891 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 213412 T891 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 213430 T891 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 214234 T891 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 214267 T891 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 214269 T891 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 214311 T891 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 214315 T891 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 214318 T891 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 214322 T891 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 214323 T891 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 214323 T891 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 214324 T891 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 214324 T891 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 214333 T891 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 214334 T891 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 214334 T891 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/control-001/cores/collection1/data/
   [junit4]   2> 214334 T891 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4642ebd
   [junit4]   2> 214335 T891 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/control-001/cores/collection1/data
   [junit4]   2> 214335 T891 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/control-001/cores/collection1/data/index/
   [junit4]   2> 214335 T891 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 214336 T891 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/control-001/cores/collection1/data/index
   [junit4]   2> 214337 T891 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1061860050, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 214338 T891 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@830a434 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@316c561f),segFN=segments_1,generation=1}
   [junit4]   2> 214338 T891 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 214343 T891 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 214343 T891 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 214343 T891 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 214343 T891 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 214344 T891 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 214344 T891 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 214344 T891 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 214345 T891 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 214345 T891 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 214359 T891 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 214361 T891 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 214363 T891 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 214365 T891 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 214369 T891 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/threads,/get,/update/json,/admin/logging,/admin/ping,/admin/luke,/admin/file,/config,/admin/plugins,/admin/mbeans,/admin/properties,/schema,/admin/system,/update/json/docs,/admin/segments,standard,/update/csv,/replication,/update
   [junit4]   2> 214371 T891 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 214372 T891 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 214373 T891 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 214374 T891 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 214374 T891 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 214375 T891 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=353497832, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 214376 T891 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@830a434 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@316c561f),segFN=segments_1,generation=1}
   [junit4]   2> 214376 T891 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 214376 T891 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@442cd9fa[collection1] main
   [junit4]   2> 214376 T891 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 214377 T891 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 214377 T891 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 214378 T891 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 214378 T891 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 214378 T891 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 214379 T891 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 214379 T891 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 214379 T891 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 214380 T891 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 214380 T891 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 214380 T892 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@442cd9fa[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 214381 T895 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:25299/jx collection:control_collection shard:shard1
   [junit4]   2> 214381 T861 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 214382 T861 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 214382 T895 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 214385 T861 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 214385 T861 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 214387 T895 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 214387 T899 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1eb69805 name:ZooKeeperConnection Watcher:127.0.0.1:25294/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 214388 T861 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 214388 T861 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 214389 T861 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 214389 T887 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 214390 T895 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 214390 T895 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 214390 T888 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C112 name=collection1 org.apache.solr.core.SolrCore@27b6bc9e url=http://127.0.0.1:25299/jx/collection1 node=127.0.0.1:25299_jx C112_STATE=coll:control_collection core:collection1 props:{state=down, node_name=127.0.0.1:25299_jx, core=collection1, base_url=http://127.0.0.1:25299/jx}
   [junit4]   2> 214391 T895 C:control_collection S:shard1 C112 P25299 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:25299/jx/collection1/
   [junit4]   2> 214391 T895 C:control_collection S:shard1 C112 P25299 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 214392 T895 C:control_collection S:shard1 C112 P25299 oasc.SyncStrategy.syncToMe http://127.0.0.1:25299/jx/collection1/ has no replicas
   [junit4]   2> 214392 T861 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 214392 T887 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 214392 T899 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 214392 T895 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:25299/jx/collection1/ shard1
   [junit4]   2> 214394 T895 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 214397 T887 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 214398 T888 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25299/jx",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 214501 T887 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 214501 T899 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 214550 T895 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:25299/jx/collection1/ and leader is http://127.0.0.1:25299/jx/collection1/
   [junit4]   2> 214551 T895 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:25299/jx
   [junit4]   2> 214551 T895 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 214551 T895 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 214551 T895 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 214553 T887 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 214554 T888 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:25299_jx",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25299/jx"} current state version: 3
   [junit4]   2> 214554 T888 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:25299_jx",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25299/jx"}
   [junit4]   2> 214658 T887 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 214660 T899 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 216145 T861 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001/cores/collection1
   [junit4]   2> 216148 T861 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001
   [junit4]   2> 216149 T861 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 216151 T861 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:60187
   [junit4]   2> 216151 T861 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=57218, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001/cores, solrconfig=solrconfig.xml, hostContext=/jx, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/tempDir-001/jetty1}
   [junit4]   2> 216153 T861 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@5fb271f1
   [junit4]   2> 216153 T861 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001/'
   [junit4]   2> 216190 T861 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001/solr.xml
   [junit4]   2> 216236 T861 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001/cores
   [junit4]   2> 216236 T861 oasc.CoreContainer.<init> New CoreContainer 1450341966
   [junit4]   2> 216237 T861 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001/]
   [junit4]   2> 216237 T861 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001/lib
   [junit4]   2> 216238 T861 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001/lib).
   [junit4]   2> 216248 T861 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 216248 T861 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 216249 T861 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 216249 T861 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 216249 T861 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 216249 T861 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 216250 T861 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 216250 T861 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 216250 T861 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 216251 T861 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 216251 T861 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 216251 T861 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 216252 T861 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 216252 T861 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 216252 T861 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 216253 T861 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:25294/solr
   [junit4]   2> 216253 T861 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 216253 T861 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 216254 T861 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 216255 T911 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@189c091e name:ZooKeeperConnection Watcher:127.0.0.1:25294 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 216256 T861 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 216257 T861 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 216259 T861 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 216260 T914 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6c40cc11 name:ZooKeeperConnection Watcher:127.0.0.1:25294/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 216260 T861 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 216264 T861 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 217267 T861 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57218_jx
   [junit4]   2> 217268 T861 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:57218_jx
   [junit4]   2> 217271 T861 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 217273 T861 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001/cores
   [junit4]   2> 217275 T861 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {loadOnStartup=true, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001/cores/collection1, collection=collection1, config=solrconfig.xml, transient=false, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001/cores/collection1/, shard=, coreNodeName=, dataDir=data/, name=collection1, schema=schema.xml}
   [junit4]   2> 217275 T861 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001/cores/collection1/
   [junit4]   2> 217275 T861 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 217277 T915 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 217277 T915 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 217278 T915 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 217278 T887 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 217278 T888 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:57218_jx",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57218/jx"} current state version: 4
   [junit4]   2> 217279 T888 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:57218_jx",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57218/jx"}
   [junit4]   2> 217279 T888 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 217279 T888 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 217280 T887 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 217280 T899 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 217280 T914 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 218278 T915 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 218279 T915 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 218280 T915 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 218280 T915 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 218280 T915 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 218281 T915 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001/cores/collection1/'
   [junit4]   2> 218296 T915 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 218299 T915 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 218313 T915 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 218336 T915 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 218390 T915 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 218392 T915 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 218411 T915 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 218777 T915 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 218793 T915 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 218795 T915 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 218808 T915 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 218813 T915 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 218817 T915 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 218818 T915 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 218818 T915 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 218818 T915 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 218819 T915 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 218819 T915 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 218819 T915 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 218820 T915 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 218820 T915 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 218820 T915 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4642ebd
   [junit4]   2> 218821 T915 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001/cores/collection1/data
   [junit4]   2> 218821 T915 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 218822 T915 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 218822 T915 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 218823 T915 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1061860050, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 218824 T915 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@252e467c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@11589f6),segFN=segments_1,generation=1}
   [junit4]   2> 218824 T915 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 218842 T915 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 218843 T915 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 218843 T915 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 218843 T915 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 218844 T915 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 218844 T915 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 218844 T915 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 218845 T915 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 218845 T915 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 218867 T915 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 218869 T915 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 218871 T915 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 218874 T915 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 218880 T915 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/threads,/get,/update/json,/admin/logging,/admin/ping,/admin/luke,/admin/file,/config,/admin/plugins,/admin/mbeans,/admin/properties,/schema,/admin/system,/update/json/docs,/admin/segments,standard,/update/csv,/replication,/update
   [junit4]   2> 218882 T915 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 218884 T915 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 218884 T915 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 218885 T915 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 218886 T915 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 218887 T915 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=353497832, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 218888 T915 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@252e467c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@11589f6),segFN=segments_1,generation=1}
   [junit4]   2> 218888 T915 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 218888 T915 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@5bd26686[collection1] main
   [junit4]   2> 218889 T915 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 218890 T915 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 218890 T915 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 218890 T915 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 218890 T915 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 218891 T915 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 218891 T915 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 218891 T915 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 218892 T915 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 218892 T915 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 218893 T916 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5bd26686[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 218893 T915 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 218894 T919 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:57218/jx collection:collection1 shard:shard1
   [junit4]   2> 218895 T861 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 218895 T861 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 218895 T919 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 218899 T919 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 218900 T887 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 218901 T919 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 218901 T919 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C113 name=collection1 org.apache.solr.core.SolrCore@1f5e0b9b url=http://127.0.0.1:57218/jx/collection1 node=127.0.0.1:57218_jx C113_STATE=coll:collection1 core:collection1 props:{state=down, node_name=127.0.0.1:57218_jx, core=collection1, base_url=http://127.0.0.1:57218/jx}
   [junit4]   2> 218901 T919 C:collection1 S:shard1 C113 P57218 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:57218/jx/collection1/
   [junit4]   2> 218901 T888 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 218902 T919 C:collection1 S:shard1 C113 P57218 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 218902 T919 C:collection1 S:shard1 C113 P57218 oasc.SyncStrategy.syncToMe http://127.0.0.1:57218/jx/collection1/ has no replicas
   [junit4]   2> 218902 T919 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:57218/jx/collection1/ shard1
   [junit4]   2> 218903 T919 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 218903 T887 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 218903 T899 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 218903 T914 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 218906 T887 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 218907 T888 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57218/jx",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 219009 T887 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 219009 T914 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 219009 T899 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 219058 T919 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:57218/jx/collection1/ and leader is http://127.0.0.1:57218/jx/collection1/
   [junit4]   2> 219059 T919 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:57218/jx
   [junit4]   2> 219059 T919 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 219059 T919 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 219060 T919 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 219061 T887 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 219063 T888 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:57218_jx",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57218/jx"} current state version: 7
   [junit4]   2> 219064 T888 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:57218_jx",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57218/jx"}
   [junit4]   2> 219217 T887 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 219217 T914 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 219217 T899 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 220628 T861 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest E7E5C63DFB521D73-001/shard-2-001/cores/colle

[...truncated too long message...]

	... 11 more
   [junit4]   2> 	
   [junit4]   2> 568795 T1939 C:control_collection S:shard1 c:collection1 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> 568975 T1939 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:48453 48453
   [junit4]   2> 570027 T1940 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:48453 48453
   [junit4]   2> 570029 T1940 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/aliases.json
   [junit4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 		3	/solr/collections/c8n_1x3_commits/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 		4	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 570029 T1939 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 1 connections to: http://127.0.0.1:48470/
   [junit4]   2> 570029 T1939 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:48487/
   [junit4]   2> 570030 T1939 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 9 connections to: http://127.0.0.1:48465/
   [junit4]   2> 570030 T1939 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:48459/
   [junit4]   2> 570031 T1939 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:48476/
   [junit4]   2> 570031 T1939 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:48470/
   [junit4]   2> 570031 T1939 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:48487/
   [junit4]   2> 570032 T1939 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:48465/
   [junit4]   2> 570032 T1939 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:48459/
   [junit4]   2> 570032 T1939 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:48476/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderInitiatedRecoveryOnCommitTest -Dtests.method=test -Dtests.seed=E7E5C63DFB521D73 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=de_CH -Dtests.timezone=America/Grenada -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   78.0s J0 | LeaderInitiatedRecoveryOnCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48476/c8n_1x3_commits_shard1_replica1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([E7E5C63DFB521D73:6FB1F9E755AE708B]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 48 more
   [junit4]   2> 570050 T1939 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E7E5C63DFB521D73-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=false,coord=no): {}, locale=de_CH, timezone=America/Grenada
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=39416776,total=329777152
   [junit4]   2> NOTE: All tests run in this JVM: [BlockDirectoryTest, TestEmbeddedSolrServerConstructors, ReturnFieldsTest, TestSchemaNameResource, DistributedQueryComponentCustomSortTest, SpellPossibilityIteratorTest, FastVectorHighlighterTest, TestDistributedMissingSort, DistributedFacetPivotLongTailTest, TestCollectionAPI, HdfsWriteToMultipleCollectionsTest, TestHighFrequencyDictionaryFactory, TestSerializedLuceneMatchVersion, VMParamsZkACLAndCredentialsProvidersTest, StressHdfsTest, BasicZkTest, BinaryUpdateRequestHandlerTest, TestCoreContainer, MergeStrategyTest, TestCoreDiscovery, TestNonDefinedSimilarityFactory, TestSuggestSpellingConverter, TestFastWriter, TestFieldResource, CloudMLTQParserTest, TestRestManager, TestIndexSearcher, DistribDocExpirationUpdateProcessorTest, SolrRequestParserTest, TestLRUCache, CustomCollectionTest, TestMacros, AutoCommitTest, TestIndexingPerformance, TestQueryUtils, SolrCoreCheckLockOnStartupTest, FieldMutatingUpdateProcessorTest, DistributedDebugComponentTest, TestJettySolrRunner, OverseerRolesTest, TestSearcherReuse, TestComplexPhraseQParserPlugin, SpellingQueryConverterTest, HdfsCollectionsAPIDistributedZkTest, OutOfBoxZkACLAndCredentialsProvidersTest, TestHashPartitioner, DateMathParserTest, PingRequestHandlerTest, LeaderInitiatedRecoveryOnCommitTest]
   [junit4] Completed on J0 in 78.02s, 1 test, 1 error <<< FAILURES!

[...truncated 1172 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 53 minutes 40 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 218897246 bytes
Compression is 0.0%
Took 41 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2873 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2873/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:59968/dw_zba/ry/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:59968/dw_zba/ry/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([AA733A9D8B345833:2227054725C835CB]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:38410/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:38410/collection1
	at __randomizedtesting.SeedInfo.seed([AA733A9D8B345833:2227054725C835CB]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:38410/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:60922/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:60922/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([AA733A9D8B345833:2227054725C835CB]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:60922/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9707 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/init-core-data-001
   [junit4]   2> 495456 T4642 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 495464 T4642 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 495465 T4643 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 495565 T4642 oasc.ZkTestServer.run start zk server on port:38396
   [junit4]   2> 495566 T4642 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 495567 T4642 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 495570 T4650 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@fb05cc0 name:ZooKeeperConnection Watcher:127.0.0.1:38396 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 495571 T4642 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 495571 T4642 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 495571 T4642 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 495574 T4642 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 495575 T4642 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 495576 T4653 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3fd658e8 name:ZooKeeperConnection Watcher:127.0.0.1:38396/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 495576 T4642 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 495577 T4642 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 495577 T4642 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 495579 T4642 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 495581 T4642 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 495582 T4642 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 495584 T4642 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 495584 T4642 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 495587 T4642 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 495587 T4642 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 495690 T4642 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 495690 T4642 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 495692 T4642 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 495693 T4642 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 495695 T4642 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 495695 T4642 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 495697 T4642 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 495698 T4642 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 495700 T4642 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 495700 T4642 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 495702 T4642 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 495702 T4642 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 495704 T4642 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 495705 T4642 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 495707 T4642 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 495707 T4642 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 495709 T4642 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 495709 T4642 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 497069 T4642 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/control-001/cores/collection1
   [junit4]   2> 497074 T4642 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 497078 T4642 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:38400
   [junit4]   2> 497079 T4642 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/control-001/cores, hostPort=38399}
   [junit4]   2> 497080 T4642 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 497080 T4642 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/control-001/'
   [junit4]   2> 497130 T4642 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/control-001/solr.xml
   [junit4]   2> 497193 T4642 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/control-001/cores
   [junit4]   2> 497193 T4642 oasc.CoreContainer.<init> New CoreContainer 350067427
   [junit4]   2> 497194 T4642 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/control-001/]
   [junit4]   2> 497194 T4642 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/control-001/lib
   [junit4]   2> 497195 T4642 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/control-001/lib).
   [junit4]   2> 497209 T4642 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 497209 T4642 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 497209 T4642 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 497210 T4642 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 497210 T4642 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 497210 T4642 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 497211 T4642 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 497211 T4642 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 497211 T4642 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 497212 T4642 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 497212 T4642 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 497213 T4642 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 497213 T4642 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 497213 T4642 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 497214 T4642 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 497214 T4642 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:38396/solr
   [junit4]   2> 497214 T4642 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 497215 T4642 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 497216 T4642 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 497217 T4665 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@62bbd71c name:ZooKeeperConnection Watcher:127.0.0.1:38396 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 497218 T4642 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 497219 T4642 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 497221 T4642 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 497222 T4668 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2e1b0f94 name:ZooKeeperConnection Watcher:127.0.0.1:38396/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 497222 T4642 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 497224 T4642 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 497226 T4642 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 497228 T4642 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 497230 T4642 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 497232 T4642 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 497235 T4642 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 497236 T4642 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 497238 T4642 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 497239 T4642 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38399_
   [junit4]   2> 497239 T4642 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:38399_
   [junit4]   2> 497241 T4642 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 497242 T4642 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 497244 T4642 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 497245 T4642 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:38399_
   [junit4]   2> 497245 T4642 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 497247 T4642 oasc.Overseer.start Overseer (id=93580187931377667-127.0.0.1:38399_-n_0000000000) starting
   [junit4]   2> 497249 T4642 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 497255 T4642 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 497256 T4670 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 497256 T4642 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 497256 T4669 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 497260 T4642 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/control-001/cores
   [junit4]   2> 497261 T4642 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/control-001/cores/collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/control-001/cores/collection1/, collection=control_collection, loadOnStartup=true, config=solrconfig.xml, coreNodeName=, shard=, dataDir=data/, name=collection1, schema=schema.xml, transient=false}
   [junit4]   2> 497262 T4642 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/control-001/cores/collection1/
   [junit4]   2> 497262 T4642 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 497263 T4672 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 497263 T4672 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 497264 T4672 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 497264 T4668 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 497265 T4669 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38399",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:38399_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1"} current state version: 0
   [junit4]   2> 497265 T4669 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38399",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:38399_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 497266 T4669 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 497266 T4669 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 497267 T4668 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 498264 T4672 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 498265 T4672 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 498265 T4672 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 498266 T4672 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 498266 T4672 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 498266 T4672 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/control-001/cores/collection1/'
   [junit4]   2> 498279 T4672 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 498281 T4672 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 498290 T4672 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 498307 T4672 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 498346 T4672 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 498447 T4672 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 498456 T4672 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 498771 T4672 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 498786 T4672 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 498788 T4672 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 498797 T4672 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 498800 T4672 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 498804 T4672 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 498804 T4672 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 498804 T4672 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 498805 T4672 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 498805 T4672 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 498805 T4672 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 498806 T4672 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 498806 T4672 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 498806 T4672 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/control-001/cores/collection1/data/
   [junit4]   2> 498806 T4672 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@71e98cd2
   [junit4]   2> 498807 T4672 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/control-001/cores/collection1/data
   [junit4]   2> 498807 T4672 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/control-001/cores/collection1/data/index/
   [junit4]   2> 498807 T4672 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 498807 T4672 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/control-001/cores/collection1/data/index
   [junit4]   2> 498808 T4672 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=140968648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 498808 T4672 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5e5fcfbc lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@643f700b),segFN=segments_1,generation=1}
   [junit4]   2> 498809 T4672 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 498812 T4672 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 498812 T4672 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 498813 T4672 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 498813 T4672 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 498813 T4672 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 498813 T4672 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 498813 T4672 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 498814 T4672 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 498814 T4672 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 498833 T4672 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 498835 T4672 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 498837 T4672 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 498838 T4672 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 498842 T4672 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /get,/admin/ping,/admin/threads,/update/json/docs,/update/csv,/admin/mbeans,/config,/admin/properties,/admin/logging,/update,standard,/update/json,/schema,/admin/file,/admin/system,/admin/luke,/admin/plugins,/admin/segments,/replication
   [junit4]   2> 498844 T4672 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 498845 T4672 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 498845 T4672 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 498847 T4672 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 498847 T4672 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 498847 T4672 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=48, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.30719632747514725]
   [junit4]   2> 498848 T4672 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5e5fcfbc lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@643f700b),segFN=segments_1,generation=1}
   [junit4]   2> 498848 T4672 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 498848 T4672 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@6a43ab06[collection1] main
   [junit4]   2> 498849 T4672 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 498849 T4672 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 498849 T4672 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 498850 T4672 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 498850 T4672 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 498850 T4672 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 498850 T4672 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 498851 T4672 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 498851 T4672 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 498851 T4672 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 498851 T4672 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 498852 T4673 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6a43ab06[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 498852 T4676 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:38399 collection:control_collection shard:shard1
   [junit4]   2> 498852 T4642 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 498853 T4642 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 498853 T4676 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 498855 T4642 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 498856 T4642 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 498857 T4676 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 498857 T4680 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@cbf4ff2 name:ZooKeeperConnection Watcher:127.0.0.1:38396/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 498857 T4642 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 498858 T4642 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 498858 T4642 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 498858 T4668 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 498859 T4676 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 498859 T4676 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C637 name=collection1 org.apache.solr.core.SolrCore@4fe7d389 url=http://127.0.0.1:38399/collection1 node=127.0.0.1:38399_ C637_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:38399, node_name=127.0.0.1:38399_, core=collection1, state=down}
   [junit4]   2> 498859 T4676 C:control_collection S:shard1 C637 P38399 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:38399/collection1/
   [junit4]   2> 498859 T4676 C:control_collection S:shard1 C637 P38399 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 498859 T4669 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 498859 T4676 C:control_collection S:shard1 C637 P38399 oasc.SyncStrategy.syncToMe http://127.0.0.1:38399/collection1/ has no replicas
   [junit4]   2> 498860 T4676 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:38399/collection1/ shard1
   [junit4]   2> 498860 T4676 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 498860 T4668 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 498860 T4680 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 498861 T4642 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 498863 T4668 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 498864 T4669 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38399",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 498967 T4668 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 498967 T4680 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 499016 T4676 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:38399/collection1/ and leader is http://127.0.0.1:38399/collection1/
   [junit4]   2> 499017 T4676 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:38399
   [junit4]   2> 499017 T4676 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 499017 T4676 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 499017 T4676 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 499018 T4668 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 499019 T4669 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38399",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:38399_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1"} current state version: 3
   [junit4]   2> 499020 T4669 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38399",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:38399_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 499122 T4680 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 499122 T4668 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 500146 T4642 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001/cores/collection1
   [junit4]   2> 500149 T4642 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001
   [junit4]   2> 500150 T4642 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 500152 T4642 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:38411
   [junit4]   2> 500152 T4642 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, hostPort=38410, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/tempDir-001/jetty1, solrconfig=solrconfig.xml}
   [junit4]   2> 500153 T4642 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 500153 T4642 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001/'
   [junit4]   2> 500200 T4642 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001/solr.xml
   [junit4]   2> 500259 T4642 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001/cores
   [junit4]   2> 500260 T4642 oasc.CoreContainer.<init> New CoreContainer 743233151
   [junit4]   2> 500260 T4642 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001/]
   [junit4]   2> 500260 T4642 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001/lib
   [junit4]   2> 500261 T4642 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001/lib).
   [junit4]   2> 500271 T4642 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 500271 T4642 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 500271 T4642 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 500271 T4642 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 500272 T4642 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 500272 T4642 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 500272 T4642 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 500272 T4642 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 500273 T4642 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 500273 T4642 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 500273 T4642 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 500273 T4642 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 500274 T4642 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 500274 T4642 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 500274 T4642 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 500274 T4642 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:38396/solr
   [junit4]   2> 500275 T4642 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 500275 T4642 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 500275 T4642 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 500277 T4692 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@710e9427 name:ZooKeeperConnection Watcher:127.0.0.1:38396 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 500277 T4642 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 500278 T4642 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 500280 T4642 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 500281 T4695 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5cbfb5cb name:ZooKeeperConnection Watcher:127.0.0.1:38396/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 500281 T4642 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 500285 T4642 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 501289 T4642 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38410_
   [junit4]   2> 501290 T4642 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:38410_
   [junit4]   2> 501292 T4642 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 501294 T4642 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001/cores
   [junit4]   2> 501297 T4642 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {dataDir=data/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001/cores/collection1, coreNodeName=, config=solrconfig.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001/cores/collection1/, collection=collection1, schema=schema.xml, loadOnStartup=true, name=collection1, transient=false, shard=}
   [junit4]   2> 501297 T4642 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001/cores/collection1/
   [junit4]   2> 501297 T4642 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 501298 T4696 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 501298 T4696 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 501299 T4668 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 501299 T4696 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 501299 T4669 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38410",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:38410_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1"} current state version: 4
   [junit4]   2> 501300 T4669 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38410",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:38410_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 501300 T4669 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 501300 T4669 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 501301 T4695 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 501301 T4680 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 501301 T4668 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 502299 T4696 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 502300 T4696 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 502301 T4696 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 502301 T4696 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 502302 T4696 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 502302 T4696 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001/cores/collection1/'
   [junit4]   2> 502314 T4696 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 502316 T4696 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 502325 T4696 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 502341 T4696 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 502403 T4696 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 502505 T4696 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 502514 T4696 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 502833 T4696 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 502854 T4696 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 502857 T4696 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 502869 T4696 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 502874 T4696 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 502878 T4696 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 502879 T4696 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 502880 T4696 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 502880 T4696 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 502881 T4696 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 502881 T4696 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 502881 T4696 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 502881 T4696 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 502882 T4696 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 502882 T4696 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@71e98cd2
   [junit4]   2> 502883 T4696 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001/cores/collection1/data
   [junit4]   2> 502883 T4696 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 502883 T4696 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 502884 T4696 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest AA733A9D8B345833-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 502885 T4696 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=140968648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 502886 T4696 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@70fec487 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6618b505),segFN=segments_1,generation=1}
   [junit4]   2> 502886 T4696 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 502892 T4696 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 502893 T4696 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 502893 T4696 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 502893 T4696 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 502893 T4696 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 502894 T4696 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 502894 T4696 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 502895 T4696 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 502895 T4696 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 502917 T4696 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 502919 T4696 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 502922 T4696 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 502925 T4696 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 502933 T4696 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /get,/admin/ping,/admin/threads,/update/json/docs,/update/csv,/admin/mbeans,/config,/admin/properties,/admin/logging,/update,standard,/update/json,/schema,/admin/file,/admin/system,/admin/luke,/admin/plugins,/admin/segments,/replication
   [junit4]   2> 502935 T4696 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 502936 T4696 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 502937 T4696 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 502938 T4696 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 502939 T4696 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 502940 T4696 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=48, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.30719632747514725]
   [junit4]   2> 502940 T4696 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@70fec487 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6618b505),segFN=segments_1,generation=1}
   [junit4]   2> 502940 T4696 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 502941 T4696 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@14dfd754[collection1] main
   [junit4]   2> 502941 T4696 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 502942 T4696 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 502943 T4696 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 502943 T4696 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 502943 T4696 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 502943 T4696 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 502944 T4696 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 502944 T4696 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 502945 T4696 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 502945 T4696 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 502946 T4697 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@14dfd754[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 502946 T4696 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 502947 T4700 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:38410 collection:collection1 shard:shard1
   [junit4]   2> 502947 T4642 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 502948 T4642 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 502948 T4700 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 502953 T4700 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 502955 T4668 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 502955 T4700 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 502955 T4700 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C638 name=collection1 org.apache.solr.core.SolrCore@5a0039e7 url=http://127.0.0.1:38410/collection1 node=127.0.0.1:38410_ C638_STATE=coll:collection1 core:collection1 props:{base_url=http://127.0.0.1:38410, node_name=127.0.0.1:38410_, core=collection1, state=down}
   [junit4]   2> 502956 T4700 C:collection1 S:shard1 C638 P38410 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:38410/collection1/
   [junit4]   2> 502956 T4700 C:collection1 S:shard1 C638 P38410 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 502956 T4669 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 502956 T4700 C:collection1 S:shard1 C638 P38410 oasc.SyncStrategy.syncToMe http://127.0.0.1:38410/collection1/ has no replicas
   [junit4]   2> 502956 T4700 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:38410/collection1/ shard1
   [junit4]   2> 502956 T4700 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 502957 T4680 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 502958 T4695 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 502958 T4668 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 502962 T4669 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38410",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 502964 T4668 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 502965 T4680 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 502965 T4695 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 502965 T4668 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 503012 T4700 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:38410/collection1/ and leader is http://127.0.0.1:38410/collection1/
   [junit4]   2> 503013 T4700 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:38410
   [junit4]   2> 503013 T4700 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 503013 T4700 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 503013 T4700 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 503015 T4668 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 503016 T4669 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38410",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:38410_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1"} current state version: 7
   [junit4]   2> 503017 T4669 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38410",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:38410_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 503121 T4680 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 503121 T4695 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDa

[...truncated too long message...]

it4]   2> 	
   [junit4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		6	/solr/aliases.json
   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 		3	/solr/collections/c8n_1x3_commits/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent data watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		2	/solr/collections/c8n_1x3_commits/leader_elect/shard1/election/93580244919779340-core_node3-n_0000000000
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 		4	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1444103 T3058 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:59968/dw_zba/ry
   [junit4]   2> 1444103 T3058 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 3 connections to: http://127.0.0.1:59877/dw_zba/ry
   [junit4]   2> 1444103 T3058 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:59993/dw_zba/ry
   [junit4]   2> 1444103 T3058 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 11 connections to: http://127.0.0.1:59886/dw_zba/ry
   [junit4]   2> 1444104 T3058 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 1 connections to: http://127.0.0.1:59938/dw_zba/ry
   [junit4]   2> 1444104 T3058 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:59968/dw_zba/ry
   [junit4]   2> 1444104 T3058 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:59877/dw_zba/ry
   [junit4]   2> 1444105 T3058 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:59993/dw_zba/ry
   [junit4]   2> 1444105 T3058 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:59886/dw_zba/ry
   [junit4]   2> 1444105 T3058 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:59938/dw_zba/ry
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderInitiatedRecoveryOnCommitTest -Dtests.method=test -Dtests.seed=AA733A9D8B345833 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=mt_MT -Dtests.timezone=Canada/Eastern -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   78.8s J1 | LeaderInitiatedRecoveryOnCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:59968/dw_zba/ry/c8n_1x3_commits_shard1_replica1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([AA733A9D8B345833:2227054725C835CB]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 48 more
   [junit4]   2> 1444126 T3058 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest AA733A9D8B345833-001
   [junit4]   2> 78834 T3057 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=true,coord=no): {}, locale=mt_MT, timezone=Canada/Eastern
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=84144608,total=299368448
   [junit4]   2> NOTE: All tests run in this JVM: [ExitableDirectoryReaderTest, TestCodecSupport, FastVectorHighlighterTest, IndexBasedSpellCheckerTest, TestCopyFieldCollectionResource, RegexBoostProcessorTest, BasicZkTest, CloudMLTQParserTest, ChaosMonkeyNothingIsSafeTest, SearchHandlerTest, TestInfoStreamLogging, TestXIncludeConfig, ScriptEngineTest, TestNRTOpen, SynonymTokenizerTest, TestConfigSets, TestSolrDynamicMBean, CopyFieldTest, MultiTermTest, URLClassifyProcessorTest, TestBulkSchemaAPI, TestBM25SimilarityFactory, ZkCLITest, TestZkChroot, TestSchemaSimilarityResource, OpenCloseCoreStressTest, DocumentBuilderTest, UpdateParamsTest, DistributedFacetPivotLargeTest, DistributedFacetPivotSmallAdvancedTest, TestPostingsSolrHighlighter, UniqFieldsUpdateProcessorFactoryTest, TestDocumentBuilder, UnloadDistributedZkTest, IndexSchemaTest, DateFieldTest, ExternalCollectionsTest, TestStressRecovery, NoCacheHeaderTest, TestJmxMonitoredMap, MBeansHandlerTest, SystemInfoHandlerTest, ResourceLoaderTest, TestJsonRequest, TestFastLRUCache, TestNamedUpdateProcessors, SimpleCollectionCreateDeleteTest, SpellingQueryConverterTest, TestBulkSchemaConcurrent, LoggingHandlerTest, TestLeaderElectionZkExpiry, RequestHandlersTest, BasicFunctionalityTest, SolrCoreTest, TestSerializedLuceneMatchVersion, TestDocSet, VMParamsZkACLAndCredentialsProvidersTest, LeaderFailoverAfterPartitionTest, BasicDistributedZk2Test, LeaderInitiatedRecoveryOnCommitTest]
   [junit4] Completed on J1 in 79.62s, 1 test, 1 error <<< FAILURES!

[...truncated 719 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 51 minutes 57 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 214555497 bytes
Compression is 0.0%
Took 37 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2872 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2872/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:46386/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:46386/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([7319EB1FBFFBB865:FB4DD4C51107D59D]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:43189/_dr/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:43189/_dr/collection1
	at __randomizedtesting.SeedInfo.seed([7319EB1FBFFBB865:FB4DD4C51107D59D]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:43189/_dr/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:57081/repfacttest_c8n_1x3_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:57081/repfacttest_c8n_1x3_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([7319EB1FBFFBB865:FB4DD4C51107D59D]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:57081/repfacttest_c8n_1x3_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9657 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/init-core-data-001
   [junit4]   2> 650391 T1557 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 650396 T1557 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 650397 T1558 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 650497 T1557 oasc.ZkTestServer.run start zk server on port:16356
   [junit4]   2> 650498 T1557 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 650499 T1557 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 650503 T1565 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@666b8451 name:ZooKeeperConnection Watcher:127.0.0.1:16356 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 650503 T1557 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 650504 T1557 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 650504 T1557 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 650507 T1557 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 650508 T1557 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 650510 T1568 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4ed528fe name:ZooKeeperConnection Watcher:127.0.0.1:16356/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 650510 T1557 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 650511 T1557 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 650511 T1557 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 650514 T1557 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 650515 T1557 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 650516 T1557 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 650517 T1557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 650518 T1557 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 650520 T1557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 650521 T1557 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 650622 T1557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 650623 T1557 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 650624 T1557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 650625 T1557 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 650626 T1557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 650626 T1557 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 650628 T1557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 650628 T1557 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 650648 T1557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 650648 T1557 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 650651 T1557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 650652 T1557 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 650657 T1557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 650658 T1557 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 650660 T1557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 650660 T1557 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 650661 T1557 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 650662 T1557 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 651604 T1557 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/control-001/cores/collection1
   [junit4]   2> 651608 T1557 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 651612 T1557 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:17992
   [junit4]   2> 651612 T1557 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/tempDir-001/control/data, hostPort=63033}
   [junit4]   2> 651613 T1557 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 651613 T1557 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/control-001/'
   [junit4]   2> 651643 T1557 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/control-001/solr.xml
   [junit4]   2> 651678 T1557 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/control-001/cores
   [junit4]   2> 651678 T1557 oasc.CoreContainer.<init> New CoreContainer 566788789
   [junit4]   2> 651678 T1557 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/control-001/]
   [junit4]   2> 651678 T1557 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/control-001/lib
   [junit4]   2> 651679 T1557 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/control-001/lib).
   [junit4]   2> 651688 T1557 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 651688 T1557 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 651688 T1557 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 651689 T1557 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 651689 T1557 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 651689 T1557 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 651689 T1557 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 651689 T1557 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 651690 T1557 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 651690 T1557 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 651690 T1557 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 651690 T1557 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 651691 T1557 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 651691 T1557 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 651691 T1557 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 651691 T1557 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:16356/solr
   [junit4]   2> 651692 T1557 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 651692 T1557 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 651693 T1557 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 651694 T1580 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7b435b90 name:ZooKeeperConnection Watcher:127.0.0.1:16356 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 651695 T1557 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 651696 T1557 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 651698 T1557 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 651699 T1583 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6c6b8fd4 name:ZooKeeperConnection Watcher:127.0.0.1:16356/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 651700 T1557 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 651701 T1557 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 651703 T1557 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 651705 T1557 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 651707 T1557 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 651708 T1557 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 651710 T1557 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 651711 T1557 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 651713 T1557 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 651714 T1557 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:63033_
   [junit4]   2> 651714 T1557 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:63033_
   [junit4]   2> 651715 T1557 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 651717 T1557 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 651718 T1557 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 651719 T1557 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:63033_
   [junit4]   2> 651720 T1557 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 651721 T1557 oasc.Overseer.start Overseer (id=93579341981417475-127.0.0.1:63033_-n_0000000000) starting
   [junit4]   2> 651722 T1557 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 651727 T1557 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 651728 T1585 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 651728 T1557 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 651728 T1584 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 651732 T1557 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/control-001/cores
   [junit4]   2> 651734 T1557 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=control_collection, coreNodeName=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/control-001/cores/collection1, name=collection1, transient=false, dataDir=data/, shard=, loadOnStartup=true, config=solrconfig.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/control-001/cores/collection1/, schema=schema.xml}
   [junit4]   2> 651734 T1557 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/control-001/cores/collection1/
   [junit4]   2> 651734 T1557 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 651735 T1587 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 651735 T1587 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 651736 T1587 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 651736 T1583 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 651737 T1584 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:63033",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:63033_",
   [junit4]   2> 	  "core":"collection1"} current state version: 0
   [junit4]   2> 651737 T1584 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:63033",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:63033_",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 651738 T1584 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 651738 T1584 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 651739 T1583 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 652736 T1587 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 652737 T1587 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 652737 T1587 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 652738 T1587 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 652738 T1587 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 652739 T1587 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/control-001/cores/collection1/'
   [junit4]   2> 652757 T1587 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 652759 T1587 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 652770 T1587 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 652792 T1587 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 652855 T1587 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 652956 T1587 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 652967 T1587 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 653375 T1587 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 653392 T1587 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 653395 T1587 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 653407 T1587 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 653412 T1587 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 653416 T1587 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 653417 T1587 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 653417 T1587 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 653418 T1587 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 653418 T1587 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 653419 T1587 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 653419 T1587 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 653419 T1587 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 653420 T1587 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/control-001/cores/collection1/data/
   [junit4]   2> 653420 T1587 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6fd80308
   [junit4]   2> 653421 T1587 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/control-001/cores/collection1/data
   [junit4]   2> 653421 T1587 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/control-001/cores/collection1/data/index/
   [junit4]   2> 653421 T1587 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 653422 T1587 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/control-001/cores/collection1/data/index
   [junit4]   2> 653422 T1587 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=37, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 653423 T1587 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@62271b45 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@756f815a),segFN=segments_1,generation=1}
   [junit4]   2> 653423 T1587 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 653430 T1587 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 653430 T1587 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 653431 T1587 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 653431 T1587 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 653431 T1587 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 653431 T1587 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 653432 T1587 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 653432 T1587 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 653433 T1587 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 653452 T1587 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 653454 T1587 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 653456 T1587 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 653458 T1587 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 653464 T1587 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: standard,/admin/file,/update/csv,/get,/update,/admin/plugins,/schema,/update/json/docs,/admin/ping,/admin/properties,/admin/logging,/admin/mbeans,/admin/system,/admin/segments,/config,/replication,/admin/threads,/admin/luke,/update/json
   [junit4]   2> 653467 T1587 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 653468 T1587 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 653468 T1587 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 653470 T1587 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 653470 T1587 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 653471 T1587 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=46, maxMergeAtOnceExplicit=49, maxMergedSegmentMB=70.00390625, floorSegmentMB=1.810546875, forceMergeDeletesPctAllowed=17.496924637359008, segmentsPerTier=16.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.19937087188091374
   [junit4]   2> 653471 T1587 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@62271b45 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@756f815a),segFN=segments_1,generation=1}
   [junit4]   2> 653472 T1587 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 653472 T1587 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@2a712cf5[collection1] main
   [junit4]   2> 653472 T1587 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 653473 T1587 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 653474 T1587 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 653474 T1587 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 653474 T1587 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 653474 T1587 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 653475 T1587 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 653475 T1587 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 653476 T1587 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 653476 T1587 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 653477 T1588 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2a712cf5[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 653477 T1587 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 653478 T1591 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:63033 collection:control_collection shard:shard1
   [junit4]   2> 653478 T1557 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 653479 T1557 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 653479 T1591 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 653482 T1557 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 653483 T1557 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 653483 T1591 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 653484 T1595 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@142d0aca name:ZooKeeperConnection Watcher:127.0.0.1:16356/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 653484 T1557 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 653485 T1583 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 653485 T1591 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 653485 T1557 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 653485 T1584 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 653485 T1557 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 653485 T1591 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C274 name=collection1 org.apache.solr.core.SolrCore@37ffd969 url=http://127.0.0.1:63033/collection1 node=127.0.0.1:63033_ C274_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:63033, state=down, node_name=127.0.0.1:63033_, core=collection1}
   [junit4]   2> 653487 T1591 C:control_collection S:shard1 C274 P63033 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:63033/collection1/
   [junit4]   2> 653487 T1591 C:control_collection S:shard1 C274 P63033 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 653487 T1583 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 653487 T1591 C:control_collection S:shard1 C274 P63033 oasc.SyncStrategy.syncToMe http://127.0.0.1:63033/collection1/ has no replicas
   [junit4]   2> 653488 T1591 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:63033/collection1/ shard1
   [junit4]   2> 653487 T1595 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 653488 T1591 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 653489 T1557 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 653492 T1583 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 653492 T1584 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:63033",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 653595 T1583 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 653596 T1595 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 653644 T1591 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:63033/collection1/ and leader is http://127.0.0.1:63033/collection1/
   [junit4]   2> 653645 T1591 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:63033
   [junit4]   2> 653645 T1591 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 653645 T1591 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 653645 T1591 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 653647 T1583 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 653647 T1584 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:63033",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:63033_",
   [junit4]   2> 	  "core":"collection1"} current state version: 3
   [junit4]   2> 653648 T1584 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:63033",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:63033_",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 653751 T1583 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 653751 T1595 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 654728 T1557 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001/cores/collection1
   [junit4]   2> 654730 T1557 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001
   [junit4]   2> 654731 T1557 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 654733 T1557 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:20037
   [junit4]   2> 654734 T1557 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/tempDir-001/jetty1, hostPort=46386, solrconfig=solrconfig.xml, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001/cores}
   [junit4]   2> 654734 T1557 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 654734 T1557 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001/'
   [junit4]   2> 654764 T1557 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001/solr.xml
   [junit4]   2> 654799 T1557 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001/cores
   [junit4]   2> 654800 T1557 oasc.CoreContainer.<init> New CoreContainer 1960725163
   [junit4]   2> 654800 T1557 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001/]
   [junit4]   2> 654800 T1557 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001/lib
   [junit4]   2> 654801 T1557 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001/lib).
   [junit4]   2> 654811 T1557 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 654811 T1557 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 654811 T1557 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 654811 T1557 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 654812 T1557 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 654812 T1557 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 654812 T1557 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 654812 T1557 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 654812 T1557 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 654813 T1557 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 654813 T1557 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 654813 T1557 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 654814 T1557 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 654814 T1557 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 654814 T1557 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 654814 T1557 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:16356/solr
   [junit4]   2> 654814 T1557 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 654815 T1557 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 654815 T1557 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 654817 T1607 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@710ed421 name:ZooKeeperConnection Watcher:127.0.0.1:16356 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 654818 T1557 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 654818 T1557 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 654821 T1557 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 654822 T1610 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@272f38c3 name:ZooKeeperConnection Watcher:127.0.0.1:16356/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 654822 T1557 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 654826 T1557 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 655829 T1557 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46386_
   [junit4]   2> 655830 T1557 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:46386_
   [junit4]   2> 655833 T1557 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 655836 T1557 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001/cores
   [junit4]   2> 655839 T1557 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001/cores/collection1/, coreNodeName=, name=collection1, schema=schema.xml, dataDir=data/, transient=false, collection=collection1, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001/cores/collection1, loadOnStartup=true, shard=}
   [junit4]   2> 655839 T1557 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001/cores/collection1/
   [junit4]   2> 655840 T1557 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 655841 T1611 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 655841 T1611 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 655842 T1611 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 655842 T1583 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 655843 T1584 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:46386",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:46386_",
   [junit4]   2> 	  "core":"collection1"} current state version: 4
   [junit4]   2> 655843 T1584 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:46386",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:46386_",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 655843 T1584 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 655844 T1584 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 655845 T1583 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 655845 T1610 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 655845 T1595 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 656842 T1611 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 656843 T1611 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 656843 T1611 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 656844 T1611 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 656844 T1611 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 656844 T1611 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001/cores/collection1/'
   [junit4]   2> 656857 T1611 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 656859 T1611 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 656867 T1611 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 656882 T1611 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 656916 T1611 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 657017 T1611 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 657024 T1611 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 657415 T1611 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 657427 T1611 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 657429 T1611 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 657438 T1611 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 657442 T1611 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 657444 T1611 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 657445 T1611 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 657445 T1611 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 657446 T1611 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 657446 T1611 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 657446 T1611 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 657447 T1611 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 657447 T1611 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 657447 T1611 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 657447 T1611 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6fd80308
   [junit4]   2> 657448 T1611 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001/cores/collection1/data
   [junit4]   2> 657448 T1611 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 657448 T1611 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 657449 T1611 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7319EB1FBFFBB865-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 657449 T1611 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=37, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 657449 T1611 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@bec9346 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@58b8dec7),segFN=segments_1,generation=1}
   [junit4]   2> 657450 T1611 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 657455 T1611 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 657456 T1611 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 657456 T1611 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 657456 T1611 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 657456 T1611 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 657456 T1611 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 657457 T1611 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 657457 T1611 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 657457 T1611 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 657469 T1611 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 657471 T1611 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 657472 T1611 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 657474 T1611 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 657480 T1611 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: standard,/admin/file,/update/csv,/get,/update,/admin/plugins,/schema,/update/json/docs,/admin/ping,/admin/properties,/admin/logging,/admin/mbeans,/admin/system,/admin/segments,/config,/replication,/admin/threads,/admin/luke,/update/json
   [junit4]   2> 657482 T1611 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 657483 T1611 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 657483 T1611 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 657484 T1611 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 657484 T1611 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 657485 T1611 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=46, maxMergeAtOnceExplicit=49, maxMergedSegmentMB=70.00390625, floorSegmentMB=1.810546875, forceMergeDeletesPctAllowed=17.496924637359008, segmentsPerTier=16.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.19937087188091374
   [junit4]   2> 657486 T1611 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@bec9346 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@58b8dec7),segFN=segments_1,generation=1}
   [junit4]   2> 657486 T1611 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 657487 T1611 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@33f8cc90[collection1] main
   [junit4]   2> 657487 T1611 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 657488 T1611 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 657488 T1611 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 657488 T1611 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 657488 T1611 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 657488 T1611 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 657489 T1611 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 657489 T1611 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 657489 T1611 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 657489 T1611 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 657491 T1612 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@33f8cc90[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 657491 T1611 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 657492 T1615 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:46386 collection:collection1 shard:shard1
   [junit4]   2> 657492 T1557 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 657493 T1557 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 657493 T1615 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 657497 T1615 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 657498 T1583 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 657498 T1615 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 657498 T1615 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C275 name=collection1 org.apache.solr.core.SolrCore@3b4dc481 url=http://127.0.0.1:46386/collection1 node=127.0.0.1:46386_ C275_STATE=coll:collection1 core:collection1 props:{base_url=http://127.0.0.1:46386, state=down, node_name=127.0.0.1:46386_, core=collection1}
   [junit4]   2> 657498 T1615 C:collection1 S:shard1 C275 P46386 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:46386/collection1/
   [junit4]   2> 657499 T1584 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 657499 T1615 C:collection1 S:shard1 C275 P46386 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 657499 T1615 C:collection1 S:shard1 C275 P46386 oasc.SyncStrategy.syncToMe http://127.0.0.1:46386/collection1/ has no replicas
   [junit4]   2> 657499 T1615 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:46386/collection1/ shard1
   [junit4]   2> 657500 T1615 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 657500 T1583 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 657500 T1610 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 657500 T1595 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 657503 T1583 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 657504 T1584 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:46386",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 657607 T1610 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 657607 T1595 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 657607 T1583 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 657656 T1615 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:46386/collection1/ and leader is http://127.0.0.1:46386/collection1/
   [junit4]   2> 657687 T1615 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:46386
   [junit4]   2> 657688 T1615 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 657688 T1615 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 657688 T1615 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 657690 T1583 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 657691 T1584 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:46386",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:46386_",
   [junit4]   2> 	  "core":"collection1"} current state version: 7
   [junit4]   2> 657691 T1584 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:46386",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:46386_",
   [junit4]   2> 	  "c

[...truncated too long message...]

tion S:shard1 c:collection1 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/_dr,null}
   [junit4]   2> 1710172 T4651 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:43174 43174
   [junit4]   2> 1710392 T4652 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:43174 43174
   [junit4]   2> 1710393 T4652 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1710394 T4651 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:43207/_dr
   [junit4]   2> 1710394 T4651 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:43189/_dr
   [junit4]   2> 1710394 T4651 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:43180/_dr
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=7319EB1FBFFBB865 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=vi -Dtests.timezone=America/Managua -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   68.3s J3 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:43189/_dr/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([7319EB1FBFFBB865:FB4DD4C51107D59D]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:43189/_dr/collection1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 49 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 54 more
   [junit4]   2> 1710410 T4651 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7319EB1FBFFBB865-001
   [junit4]   2> 68280 T4650 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50), sim=RandomSimilarityProvider(queryNorm=true,coord=no): {}, locale=vi, timezone=America/Managua
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=93747856,total=325582848
   [junit4]   2> NOTE: All tests run in this JVM: [NumericFieldsTest, SuggesterTSTTest, TestNamedUpdateProcessors, DistributedExpandComponentTest, TestHashPartitioner, BasicDistributedZk2Test, RankQueryTest, TestIntervalFaceting, TestConfigOverlay, TestDefaultSearchFieldResource, DeleteInactiveReplicaTest, TestMissingGroups, CurrencyFieldOpenExchangeTest, BlockCacheTest, XsltUpdateRequestHandlerTest, AnalyticsQueryTest, FileBasedSpellCheckerTest, TestWordDelimiterFilterFactory, MigrateRouteKeyTest, HdfsBasicDistributedZkTest, TestExtendedDismaxParser, PingRequestHandlerTest, TestNonDefinedSimilarityFactory, TestExceedMaxTermLength, QueryResultKeyTest, LeaderInitiatedRecoveryOnCommitTest, TestJsonRequest, FastVectorHighlighterTest, TestQuerySenderListener, IndexBasedSpellCheckerTest, TestFieldTypeResource, TestPivotHelperCode, ExternalCollectionsTest, TestRangeQuery, TestXIncludeConfig, ZkSolrClientTest, HdfsDirectoryFactoryTest, SpellingQueryConverterTest, TestJmxIntegration, TestExactSharedStatsCache, TestIndexingPerformance, HighlighterTest, SOLR749Test, UniqFieldsUpdateProcessorFactoryTest, LukeRequestHandlerTest, TestSearchPerf, XmlUpdateRequestHandlerTest, TestDistributedMissingSort, TestReload, DirectSolrConnectionTest, SolrXmlInZkTest, TestArbitraryIndexDir, TestBulkSchemaConcurrent, TestAddFieldRealTimeGet, HdfsChaosMonkeySafeLeaderTest, SyncSliceTest, ShardSplitTest, FullSolrCloudDistribCmdsTest, TestSolrConfigHandlerConcurrent, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J3 in 68.56s, 1 test, 1 error <<< FAILURES!

[...truncated 689 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 55 minutes 8 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 224382140 bytes
Compression is 0.0%
Took 40 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2871 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2871/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:28021/jx_pe/gu/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28021/jx_pe/gu/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([634A0386B04539CD:EB1E3C5C1EB95435]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28336/jt_tm/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28336/jt_tm/collection1
	at __randomizedtesting.SeedInfo.seed([634A0386B04539CD:EB1E3C5C1EB95435]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28336/jt_tm/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:41932/repfacttest_c8n_1x3_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:41932/repfacttest_c8n_1x3_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([634A0386B04539CD:EB1E3C5C1EB95435]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:41932/repfacttest_c8n_1x3_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9626 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/init-core-data-001
   [junit4]   2> 364503 T1514 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 364508 T1514 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 364508 T1515 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 364608 T1514 oasc.ZkTestServer.run start zk server on port:41901
   [junit4]   2> 364609 T1514 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 364610 T1514 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 364613 T1522 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6133293f name:ZooKeeperConnection Watcher:127.0.0.1:41901 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 364613 T1514 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 364614 T1514 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 364614 T1514 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 364616 T1514 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 364617 T1514 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 364618 T1525 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2ebd36bd name:ZooKeeperConnection Watcher:127.0.0.1:41901/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 364618 T1514 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 364618 T1514 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 364618 T1514 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 364620 T1514 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 364621 T1514 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 364622 T1514 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 364623 T1514 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 364624 T1514 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 364626 T1514 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 364626 T1514 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 364628 T1514 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 364629 T1514 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 364630 T1514 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 364630 T1514 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 364632 T1514 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 364632 T1514 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 364633 T1514 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 364634 T1514 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 364635 T1514 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 364635 T1514 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 364637 T1514 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 364637 T1514 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 364639 T1514 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 364639 T1514 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 364640 T1514 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 364641 T1514 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 364642 T1514 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 364642 T1514 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 365603 T1514 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/control-001/cores/collection1
   [junit4]   2> 365606 T1514 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 365610 T1514 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:41912
   [junit4]   2> 365610 T1514 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=41911, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/control-001/cores}
   [junit4]   2> 365611 T1514 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 365611 T1514 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/control-001/'
   [junit4]   2> 365644 T1514 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/control-001/solr.xml
   [junit4]   2> 365692 T1514 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/control-001/cores
   [junit4]   2> 365692 T1514 oasc.CoreContainer.<init> New CoreContainer 2074659216
   [junit4]   2> 365692 T1514 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/control-001/]
   [junit4]   2> 365693 T1514 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/control-001/lib
   [junit4]   2> 365693 T1514 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/control-001/lib).
   [junit4]   2> 365702 T1514 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 365702 T1514 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 365702 T1514 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 365703 T1514 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 365703 T1514 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 365703 T1514 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 365703 T1514 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 365703 T1514 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 365704 T1514 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 365704 T1514 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 365704 T1514 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 365705 T1514 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 365705 T1514 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 365705 T1514 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 365705 T1514 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 365706 T1514 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:41901/solr
   [junit4]   2> 365706 T1514 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 365706 T1514 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 365707 T1514 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 365709 T1537 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6142d975 name:ZooKeeperConnection Watcher:127.0.0.1:41901 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 365709 T1514 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 365710 T1514 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 365712 T1514 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 365713 T1540 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@475802dc name:ZooKeeperConnection Watcher:127.0.0.1:41901/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 365713 T1514 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 365714 T1514 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 365716 T1514 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 365718 T1514 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 365719 T1514 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 365721 T1514 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 365723 T1514 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 365724 T1514 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 365725 T1514 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 365726 T1514 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41911_
   [junit4]   2> 365727 T1514 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:41911_
   [junit4]   2> 365728 T1514 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 365729 T1514 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 365731 T1514 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 365732 T1514 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:41911_
   [junit4]   2> 365732 T1514 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 365733 T1514 oasc.Overseer.start Overseer (id=93578256990404611-127.0.0.1:41911_-n_0000000000) starting
   [junit4]   2> 365735 T1514 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 365740 T1514 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 365741 T1542 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 365741 T1514 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 365741 T1541 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 365744 T1514 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/control-001/cores
   [junit4]   2> 365746 T1514 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {transient=false, dataDir=data/, shard=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/control-001/cores/collection1/, coreNodeName=, schema=schema.xml, collection=control_collection, name=collection1, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/control-001/cores/collection1, loadOnStartup=true}
   [junit4]   2> 365746 T1514 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/control-001/cores/collection1/
   [junit4]   2> 365747 T1514 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 365747 T1544 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 365747 T1544 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 365748 T1544 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 365748 T1540 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 365749 T1541 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41911",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:41911_"} current state version: 0
   [junit4]   2> 365749 T1541 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41911",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:41911_"}
   [junit4]   2> 365750 T1541 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 365750 T1541 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 365751 T1540 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 366748 T1544 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 366749 T1544 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 366749 T1544 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 366750 T1544 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 366750 T1544 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 366751 T1544 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/control-001/cores/collection1/'
   [junit4]   2> 366767 T1544 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 366770 T1544 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 366780 T1544 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 366803 T1544 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 366852 T1544 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 366854 T1544 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 366865 T1544 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 367309 T1544 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 367326 T1544 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 367329 T1544 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 367344 T1544 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 367349 T1544 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 367353 T1544 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 367353 T1544 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 367354 T1544 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 367354 T1544 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 367355 T1544 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 367355 T1544 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 367355 T1544 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 367356 T1544 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 367356 T1544 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/control-001/cores/collection1/data/
   [junit4]   2> 367356 T1544 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5143bf4
   [junit4]   2> 367357 T1544 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/control-001/cores/collection1/data
   [junit4]   2> 367358 T1544 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/control-001/cores/collection1/data/index/
   [junit4]   2> 367358 T1544 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 367358 T1544 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/control-001/cores/collection1/data/index
   [junit4]   2> 367359 T1544 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=10, maxMergeAtOnceExplicit=47, maxMergedSegmentMB=12.9931640625, floorSegmentMB=0.5966796875, forceMergeDeletesPctAllowed=21.044805038717328, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 367360 T1544 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6718d7c5 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@ffdfb64),segFN=segments_1,generation=1}
   [junit4]   2> 367360 T1544 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 367367 T1544 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 367367 T1544 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 367368 T1544 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 367368 T1544 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 367368 T1544 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 367368 T1544 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 367369 T1544 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 367369 T1544 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 367369 T1544 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 367388 T1544 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 367391 T1544 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 367393 T1544 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 367395 T1544 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 367401 T1544 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/logging,/admin/luke,/admin/ping,/get,/update/json/docs,/config,/admin/segments,/update/csv,/admin/mbeans,/update/json,/admin/system,/update,/admin/properties,/admin/plugins,/replication,standard,/admin/file,/schema,/admin/threads
   [junit4]   2> 367403 T1544 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 367405 T1544 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 367405 T1544 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 367407 T1544 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 367407 T1544 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 367408 T1544 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=15, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=61.974609375, floorSegmentMB=0.5615234375, forceMergeDeletesPctAllowed=4.815556489907208, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.445132059291366
   [junit4]   2> 367408 T1544 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6718d7c5 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@ffdfb64),segFN=segments_1,generation=1}
   [junit4]   2> 367409 T1544 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 367409 T1544 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@47d821bf[collection1] main
   [junit4]   2> 367409 T1544 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 367410 T1544 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 367410 T1544 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 367411 T1544 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 367411 T1544 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 367411 T1544 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 367412 T1544 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 367412 T1544 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 367412 T1544 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 367413 T1544 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 367414 T1545 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@47d821bf[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 367414 T1544 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 367415 T1548 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:41911 collection:control_collection shard:shard1
   [junit4]   2> 367415 T1514 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 367416 T1514 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 367416 T1548 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 367419 T1514 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 367419 T1514 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 367421 T1552 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6c3b7305 name:ZooKeeperConnection Watcher:127.0.0.1:41901/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 367421 T1548 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 367421 T1514 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 367422 T1514 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 367422 T1514 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 367423 T1540 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 367423 T1548 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 367423 T1548 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C220 name=collection1 org.apache.solr.core.SolrCore@7d794040 url=http://127.0.0.1:41911/collection1 node=127.0.0.1:41911_ C220_STATE=coll:control_collection core:collection1 props:{state=down, core=collection1, base_url=http://127.0.0.1:41911, node_name=127.0.0.1:41911_}
   [junit4]   2> 367423 T1548 C:control_collection S:shard1 C220 P41911 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:41911/collection1/
   [junit4]   2> 367423 T1541 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 367424 T1548 C:control_collection S:shard1 C220 P41911 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 367424 T1548 C:control_collection S:shard1 C220 P41911 oasc.SyncStrategy.syncToMe http://127.0.0.1:41911/collection1/ has no replicas
   [junit4]   2> 367424 T1548 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:41911/collection1/ shard1
   [junit4]   2> 367425 T1548 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 367425 T1514 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 367425 T1552 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 367425 T1540 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 367429 T1540 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 367429 T1541 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41911",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 367532 T1540 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 367532 T1552 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 367581 T1548 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:41911/collection1/ and leader is http://127.0.0.1:41911/collection1/
   [junit4]   2> 367582 T1548 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:41911
   [junit4]   2> 367582 T1548 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 367582 T1548 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 367583 T1548 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 367584 T1540 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 367585 T1541 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41911",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:41911_"} current state version: 3
   [junit4]   2> 367586 T1541 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41911",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:41911_"}
   [junit4]   2> 367689 T1552 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 367689 T1540 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 368343 T1514 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001/cores/collection1
   [junit4]   2> 368346 T1514 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001
   [junit4]   2> 368347 T1514 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 368349 T1514 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:41929
   [junit4]   2> 368349 T1514 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, hostPort=41928, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/tempDir-001/jetty1, solrconfig=solrconfig.xml}
   [junit4]   2> 368350 T1514 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 368350 T1514 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001/'
   [junit4]   2> 368379 T1514 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001/solr.xml
   [junit4]   2> 368424 T1514 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001/cores
   [junit4]   2> 368424 T1514 oasc.CoreContainer.<init> New CoreContainer 1311768164
   [junit4]   2> 368425 T1514 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001/]
   [junit4]   2> 368425 T1514 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001/lib
   [junit4]   2> 368426 T1514 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001/lib).
   [junit4]   2> 368438 T1514 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 368439 T1514 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 368439 T1514 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 368439 T1514 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 368440 T1514 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 368440 T1514 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 368440 T1514 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 368441 T1514 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 368441 T1514 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 368441 T1514 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 368442 T1514 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 368442 T1514 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 368443 T1514 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 368443 T1514 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 368443 T1514 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 368444 T1514 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:41901/solr
   [junit4]   2> 368444 T1514 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 368444 T1514 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 368445 T1514 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 368447 T1564 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3dbf589a name:ZooKeeperConnection Watcher:127.0.0.1:41901 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 368447 T1514 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 368448 T1514 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 368450 T1514 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 368452 T1567 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6745ac53 name:ZooKeeperConnection Watcher:127.0.0.1:41901/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 368452 T1514 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 368457 T1514 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 369461 T1514 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41928_
   [junit4]   2> 369461 T1514 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:41928_
   [junit4]   2> 369464 T1514 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 369466 T1514 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001/cores
   [junit4]   2> 369468 T1514 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {transient=false, config=solrconfig.xml, collection=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001/cores/collection1, dataDir=data/, coreNodeName=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001/cores/collection1/, name=collection1, schema=schema.xml, loadOnStartup=true, shard=}
   [junit4]   2> 369468 T1514 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001/cores/collection1/
   [junit4]   2> 369468 T1514 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 369469 T1568 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 369469 T1568 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 369470 T1540 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 369470 T1568 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 369471 T1541 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41928",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:41928_"} current state version: 4
   [junit4]   2> 369472 T1541 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41928",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:41928_"}
   [junit4]   2> 369472 T1541 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 369472 T1541 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 369473 T1567 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 369473 T1552 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 369473 T1540 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 370471 T1568 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 370472 T1568 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 370472 T1568 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 370473 T1568 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 370473 T1568 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 370474 T1568 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001/cores/collection1/'
   [junit4]   2> 370490 T1568 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 370493 T1568 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 370503 T1568 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 370525 T1568 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 370573 T1568 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 370575 T1568 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 370586 T1568 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 370918 T1568 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 370930 T1568 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 370932 T1568 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 370943 T1568 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 370946 T1568 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 370948 T1568 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 370949 T1568 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 370949 T1568 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 370949 T1568 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 370950 T1568 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 370950 T1568 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 370950 T1568 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 370951 T1568 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 370951 T1568 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 370951 T1568 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5143bf4
   [junit4]   2> 370952 T1568 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001/cores/collection1/data
   [junit4]   2> 370952 T1568 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 370952 T1568 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 370952 T1568 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 634A0386B04539CD-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 370953 T1568 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=10, maxMergeAtOnceExplicit=47, maxMergedSegmentMB=12.9931640625, floorSegmentMB=0.5966796875, forceMergeDeletesPctAllowed=21.044805038717328, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 370953 T1568 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1bfbd325 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@31e3307d),segFN=segments_1,generation=1}
   [junit4]   2> 370954 T1568 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 370959 T1568 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 370959 T1568 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 370960 T1568 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 370960 T1568 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 370960 T1568 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 370960 T1568 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 370961 T1568 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 370961 T1568 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 370961 T1568 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 370973 T1568 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 370975 T1568 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 370976 T1568 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 370978 T1568 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 370984 T1568 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/logging,/admin/luke,/admin/ping,/get,/update/json/docs,/config,/admin/segments,/update/csv,/admin/mbeans,/update/json,/admin/system,/update,/admin/properties,/admin/plugins,/replication,standard,/admin/file,/schema,/admin/threads
   [junit4]   2> 370985 T1568 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 370987 T1568 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 370987 T1568 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 370988 T1568 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 370988 T1568 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 370989 T1568 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=15, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=61.974609375, floorSegmentMB=0.5615234375, forceMergeDeletesPctAllowed=4.815556489907208, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.445132059291366
   [junit4]   2> 370990 T1568 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1bfbd325 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@31e3307d),segFN=segments_1,generation=1}
   [junit4]   2> 370990 T1568 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 370990 T1568 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@27f027d1[collection1] main
   [junit4]   2> 370990 T1568 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 370991 T1568 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 370991 T1568 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 370992 T1568 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 370992 T1568 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 370992 T1568 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 370992 T1568 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 370993 T1568 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 370993 T1568 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 370993 T1568 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 370994 T1569 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@27f027d1[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 370995 T1568 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 370995 T1572 C:collection1 S:shard2 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:41928 collection:collection1 shard:shard2
   [junit4]   2> 370996 T1514 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 370996 T1514 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 370996 T1572 C:collection1 S:shard2 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 371000 T1572 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 371001 T1540 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 371001 T1572 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 371001 T1572 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C221 name=collection1 org.apache.solr.core.SolrCore@78cbe4af url=http://127.0.0.1:41928/collection1 node=127.0.0.1:41928_ C221_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, base_url=http://127.0.0.1:41928, node_name=127.0.0.1:41928_}
   [junit4]   2> 371002 T1572 C:collection1 S:shard2 C221 P41928 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:41928/collection1/
   [junit4]   2> 371002 T1541 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 371002 T1572 C:collection1 S:shard2 C221 P41928 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 371003 T1572 C:collection1 S:shard2 C221 P41928 oasc.SyncStrategy.syncToMe http://127.0.0.1:41928/collection1/ has no replicas
   [junit4]   2> 371003 T1572 C:collection1 S:shard2 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:41928/collection1/ shard2
   [junit4]   2> 371003 T1572 C:collection1 S:shard2 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 371004 T1540 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 371004 T1552 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 371004 T1567 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 371007 T1540 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 371007 T1541 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41928",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 371110 T1540 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 371110 T1552 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 371111 T1567 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 371159 T1572 C:collection1 S:shard2 oasc.ZkController.register We are http://127.0.0.1:41928/collection1/ and leader is http://127.0.0.1:41928/collection1/
   [junit4]   2> 371160 T1572 C:collection1 S:shard2 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:41928
   [junit4]   2> 371160 T1572 C:collection1 S:shard2 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 371160 T1572 C:collection1 S:shard2 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 371160 T1572 C:collection1 S:shard2 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 371162 T1540 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 371162 T1541 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41928",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:41928_"} current state version: 7
   [junit4]   2> 371163 T1541 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41928",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:41928_"}
   [junit4]   2> 371266 T1552 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 371266 T1540 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 371266 T1567 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataCha

[...truncated too long message...]

index.20150401144121430
   [junit4]   2> 1611851 T7122 C:control_collection S:shard1 c:collection1 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/jt_tm,null}
   [junit4]   2> 1612028 T7122 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:28319 28319
   [junit4]   2> 1612381 T7123 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:28319 28319
   [junit4]   2> 1612382 T7123 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1612382 T7122 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:28325/jt_tm
   [junit4]   2> 1612382 T7122 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:28354/jt_tm
   [junit4]   2> 1612383 T7122 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 6 connections to: http://127.0.0.1:28336/jt_tm
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=634A0386B04539CD -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ar_SY -Dtests.timezone=Africa/Kigali -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   71.0s J1 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28336/jt_tm/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([634A0386B04539CD:EB1E3C5C1EB95435]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28336/jt_tm/collection1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 49 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 54 more
   [junit4]   2> 1612404 T7122 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 634A0386B04539CD-001
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {}, locale=ar_SY, timezone=Africa/Kigali
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=227733656,total=382730240
   [junit4]   2> NOTE: All tests run in this JVM: [TestPerFieldSimilarity, SuggesterTSTTest, TestNamedUpdateProcessors, DocValuesTest, MergeStrategyTest, HdfsChaosMonkeySafeLeaderTest, TestSchemaVersionResource, TestSchemaSimilarityResource, LoggingHandlerTest, NotRequiredUniqueKeyTest, SolrTestCaseJ4Test, DateFieldTest, StandardRequestHandlerTest, MinimalSchemaTest, AnalyticsMergeStrategyTest, AnalyticsQueryTest, TestIntervalFaceting, TestFiltering, SortByFunctionTest, ClusterStateTest, DistributedFacetPivotLongTailTest, DistributedSpellCheckComponentTest, TestBinaryField, AutoCommitTest, TestJmxIntegration, DisMaxRequestHandlerTest, TestUniqueKeyFieldResource, MoreLikeThisHandlerTest, CoreAdminCreateDiscoverTest, TestSystemIdResolver, TestSolrQueryParserDefaultOperatorResource, TestFoldingMultitermQuery, TestMaxScoreQueryParser, IndexSchemaTest, SpatialFilterTest, ChangedSchemaMergeTest, TestRestManager, SegmentsInfoRequestHandlerTest, SolrIndexConfigTest, TestFastLRUCache, DistributedFacetPivotWhiteBoxTest, TestJoin, TestManagedResourceStorage, TestSolrConfigHandlerConcurrent, SoftAutoCommitTest, TestFieldCollectionResource, OutputWriterTest, DeleteShardTest, DistribCursorPagingTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, SampleTest, BasicDistributedZkTest, AliasIntegrationTest, SpellingQueryConverterTest, DistanceUnitsTest, PathHierarchyTokenizerFactoryTest, SuggesterTest, LeaderFailoverAfterPartitionTest, TestReplicationHandler, TestCloudSchemaless, TestSolrConfigHandlerCloud, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J1 in 71.01s, 1 test, 1 error <<< FAILURES!

[...truncated 709 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 53 minutes 58 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 211947746 bytes
Compression is 0.0%
Took 41 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2870 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2870/

4 tests failed.
REGRESSION:  org.apache.solr.handler.component.DistributedMLTComponentTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:55514/k_/e/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:55514/k_/e/collection1
	at __randomizedtesting.SeedInfo.seed([79C53BF88934A731:F191042227C8CAC9]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:566)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:943)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:558)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:606)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:588)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:567)
	at org.apache.solr.handler.component.DistributedMLTComponentTest.test(DistributedMLTComponentTest.java:126)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 51 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:50113/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:50113/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([79C53BF88934A731:F191042227C8CAC9]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([79C53BF88934A731:F191042227C8CAC9]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([79C53BF88934A731:F191042227C8CAC9]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9493 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/init-core-data-001
   [junit4]   2> 405662 T802 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 405667 T802 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 405668 T803 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 405768 T802 oasc.ZkTestServer.run start zk server on port:53026
   [junit4]   2> 405769 T802 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 405770 T802 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 405776 T810 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@27dfc329 name:ZooKeeperConnection Watcher:127.0.0.1:53026 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 405776 T802 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 405777 T802 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 405777 T802 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 405781 T802 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 405782 T802 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 405786 T813 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6115fc9b name:ZooKeeperConnection Watcher:127.0.0.1:53026/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 405786 T802 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 405787 T802 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 405787 T802 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 405790 T802 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 405792 T802 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 405793 T802 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 405794 T802 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 405795 T802 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 405797 T802 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 405798 T802 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 405900 T802 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 405901 T802 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 405902 T802 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 405902 T802 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 405904 T802 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 405904 T802 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 405906 T802 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 405906 T802 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 405907 T802 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 405908 T802 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 405909 T802 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 405909 T802 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 405911 T802 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 405911 T802 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 405912 T802 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 405913 T802 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 405914 T802 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 405914 T802 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 407291 T802 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/control-001/cores/collection1
   [junit4]   2> 407295 T802 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 407314 T802 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:53033
   [junit4]   2> 407315 T802 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, hostPort=53032, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/control-001/cores}
   [junit4]   2> 407315 T802 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 407316 T802 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/control-001/'
   [junit4]   2> 407364 T802 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/control-001/solr.xml
   [junit4]   2> 407434 T802 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/control-001/cores
   [junit4]   2> 407435 T802 oasc.CoreContainer.<init> New CoreContainer 1159902004
   [junit4]   2> 407436 T802 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/control-001/]
   [junit4]   2> 407436 T802 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/control-001/lib
   [junit4]   2> 407437 T802 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/control-001/lib).
   [junit4]   2> 407450 T802 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 407451 T802 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 407451 T802 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 407451 T802 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 407451 T802 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 407452 T802 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 407452 T802 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 407452 T802 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 407453 T802 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 407453 T802 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 407453 T802 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 407454 T802 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 407455 T802 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 407455 T802 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 407456 T802 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 407456 T802 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:53026/solr
   [junit4]   2> 407456 T802 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 407457 T802 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 407457 T802 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 407459 T825 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@319feebf name:ZooKeeperConnection Watcher:127.0.0.1:53026 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 407461 T802 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 407462 T802 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 407467 T802 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 407470 T828 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2405215f name:ZooKeeperConnection Watcher:127.0.0.1:53026/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 407471 T802 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 407474 T802 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 407476 T802 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 407480 T802 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 407488 T802 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 407490 T802 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 407494 T802 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 407496 T802 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 407498 T802 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 407500 T802 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53032_
   [junit4]   2> 407544 T802 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:53032_
   [junit4]   2> 407546 T802 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 407547 T802 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 407548 T802 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 407553 T802 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:53032_
   [junit4]   2> 407554 T802 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 407556 T802 oasc.Overseer.start Overseer (id=93577315875356675-127.0.0.1:53032_-n_0000000000) starting
   [junit4]   2> 407559 T802 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 407568 T802 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 407569 T830 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 407569 T802 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 407571 T829 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 407575 T802 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/control-001/cores
   [junit4]   2> 407578 T802 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=control_collection, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/control-001/cores/collection1/, coreNodeName=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/control-001/cores/collection1, dataDir=data/, schema=schema.xml, config=solrconfig.xml, shard=, loadOnStartup=true, transient=false, name=collection1}
   [junit4]   2> 407579 T802 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/control-001/cores/collection1/
   [junit4]   2> 407579 T802 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 407581 T832 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 407581 T832 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 407582 T832 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 407582 T828 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 407583 T829 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:53032",
   [junit4]   2> 	  "node_name":"127.0.0.1:53032_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection"} current state version: 0
   [junit4]   2> 407584 T829 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:53032",
   [junit4]   2> 	  "node_name":"127.0.0.1:53032_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection"}
   [junit4]   2> 407585 T829 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 407585 T829 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 407587 T828 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 408583 T832 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 408584 T832 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 408585 T832 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 408585 T832 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 408586 T832 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 408587 T832 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/control-001/cores/collection1/'
   [junit4]   2> 408611 T832 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 408614 T832 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 408627 T832 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 408655 T832 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 408708 T832 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 408810 T832 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 408826 T832 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 409334 T832 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 409354 T832 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 409358 T832 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 409380 T832 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 409388 T832 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 409397 T832 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 409398 T832 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 409399 T832 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 409399 T832 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 409400 T832 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 409401 T832 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 409401 T832 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 409402 T832 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 409402 T832 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/control-001/cores/collection1/data/
   [junit4]   2> 409403 T832 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@30f6bbb3
   [junit4]   2> 409422 T832 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/control-001/cores/collection1/data
   [junit4]   2> 409422 T832 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/control-001/cores/collection1/data/index/
   [junit4]   2> 409423 T832 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 409423 T832 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/control-001/cores/collection1/data/index
   [junit4]   2> 409424 T832 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=47, maxMergeAtOnceExplicit=39, maxMergedSegmentMB=91.076171875, floorSegmentMB=1.6279296875, forceMergeDeletesPctAllowed=16.892053792183308, segmentsPerTier=25.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 409425 T832 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3faae744 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6d354203),segFN=segments_1,generation=1}
   [junit4]   2> 409426 T832 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 409436 T832 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 409436 T832 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 409437 T832 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 409437 T832 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 409438 T832 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 409438 T832 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 409439 T832 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 409439 T832 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 409440 T832 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 409468 T832 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 409472 T832 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 409474 T832 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 409476 T832 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 409486 T832 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/system,/admin/file,/admin/logging,/schema,standard,/update/json,/admin/properties,/admin/ping,/config,/update/csv,/admin/threads,/admin/segments,/update,/admin/plugins,/admin/luke,/update/json/docs,/admin/mbeans,/replication,/get
   [junit4]   2> 409488 T832 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 409490 T832 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 409490 T832 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 409492 T832 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 409493 T832 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 409494 T832 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=31, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=10.6259765625, floorSegmentMB=1.404296875, forceMergeDeletesPctAllowed=27.440015656443567, segmentsPerTier=35.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 409495 T832 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3faae744 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6d354203),segFN=segments_1,generation=1}
   [junit4]   2> 409495 T832 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 409496 T832 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@bffbad6[collection1] main
   [junit4]   2> 409496 T832 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 409497 T832 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 409497 T832 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 409498 T832 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 409498 T832 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 409499 T832 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 409500 T832 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 409500 T832 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 409501 T832 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 409501 T832 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 409502 T833 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@bffbad6[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 409503 T832 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 409504 T836 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:53032 collection:control_collection shard:shard1
   [junit4]   2> 409505 T802 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 409505 T802 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 409506 T836 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 409510 T802 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 409511 T802 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 409514 T840 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@387d3813 name:ZooKeeperConnection Watcher:127.0.0.1:53026/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 409514 T836 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 409514 T802 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 409516 T802 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 409516 T802 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 409517 T828 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 409518 T836 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 409519 T836 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 409519 T829 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C76 name=collection1 org.apache.solr.core.SolrCore@18f10812 url=http://127.0.0.1:53032/collection1 node=127.0.0.1:53032_ C76_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:53032, node_name=127.0.0.1:53032_, core=collection1, state=down}
   [junit4]   2> 409519 T836 C:control_collection S:shard1 C76 P53032 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:53032/collection1/
   [junit4]   2> 409520 T836 C:control_collection S:shard1 C76 P53032 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 409521 T836 C:control_collection S:shard1 C76 P53032 oasc.SyncStrategy.syncToMe http://127.0.0.1:53032/collection1/ has no replicas
   [junit4]   2> 409521 T836 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:53032/collection1/ shard1
   [junit4]   2> 409522 T828 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 409522 T836 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 409522 T840 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 409522 T802 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 409525 T802 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 409525 T802 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 409526 T802 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 409528 T843 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@ac37ab2 name:ZooKeeperConnection Watcher:127.0.0.1:53026/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 409529 T802 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 409530 T802 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 409531 T828 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 409533 T829 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:53032",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 409538 T829 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 409538 T829 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 409540 T828 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 409540 T840 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 409550 T828 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 409552 T829 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 409554 T840 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 409554 T828 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 409583 T836 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:53032/collection1/ and leader is http://127.0.0.1:53032/collection1/
   [junit4]   2> 409584 T836 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:53032
   [junit4]   2> 409584 T836 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 409585 T836 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 409585 T836 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 409587 T828 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 409589 T829 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:53032",
   [junit4]   2> 	  "node_name":"127.0.0.1:53032_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection"} current state version: 4
   [junit4]   2> 409589 T829 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:53032",
   [junit4]   2> 	  "node_name":"127.0.0.1:53032_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection"}
   [junit4]   2> 409695 T840 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 409695 T828 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 410994 T802 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/shard-1-001/cores/collection1
   [junit4]   2> 410997 T802 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/shard-1-001
   [junit4]   2> 410998 T802 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 411000 T802 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:53053
   [junit4]   2> 411001 T802 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/tempDir-001/jetty1, hostContext=/, solrconfig=solrconfig.xml, hostPort=53052, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/shard-1-001/cores}
   [junit4]   2> 411001 T802 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 411002 T802 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/shard-1-001/'
   [junit4]   2> 411039 T802 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/shard-1-001/solr.xml
   [junit4]   2> 411098 T802 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/shard-1-001/cores
   [junit4]   2> 411098 T802 oasc.CoreContainer.<init> New CoreContainer 1323082995
   [junit4]   2> 411099 T802 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/shard-1-001/]
   [junit4]   2> 411099 T802 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/shard-1-001/lib
   [junit4]   2> 411100 T802 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/shard-1-001/lib).
   [junit4]   2> 411112 T802 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 411112 T802 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 411112 T802 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 411113 T802 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 411113 T802 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 411113 T802 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 411113 T802 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 411114 T802 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 411114 T802 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 411114 T802 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 411114 T802 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 411115 T802 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 411115 T802 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 411116 T802 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 411116 T802 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 411116 T802 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:53026/solr
   [junit4]   2> 411116 T802 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 411117 T802 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 411117 T802 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 411119 T855 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2fcb4bce name:ZooKeeperConnection Watcher:127.0.0.1:53026 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 411120 T802 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 411121 T802 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 411123 T802 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 411124 T858 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@69e58736 name:ZooKeeperConnection Watcher:127.0.0.1:53026/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 411124 T802 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 411128 T802 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 412132 T802 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53052_
   [junit4]   2> 412133 T802 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:53052_
   [junit4]   2> 412136 T802 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 412138 T802 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/shard-1-001/cores
   [junit4]   2> 412140 T802 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/shard-1-001/cores/collection1, collection=collection1, coreNodeName=, schema=schema.xml, shard=, dataDir=data/, loadOnStartup=true, config=solrconfig.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/shard-1-001/cores/collection1/, transient=false}
   [junit4]   2> 412140 T802 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/shard-1-001/cores/collection1/
   [junit4]   2> 412141 T802 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 412142 T859 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 412143 T859 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 412144 T828 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 412144 T859 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 412144 T859 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 412145 T829 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:53052",
   [junit4]   2> 	  "node_name":"127.0.0.1:53052_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 412145 T859 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 412145 T829 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:53052",
   [junit4]   2> 	  "node_name":"127.0.0.1:53052_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1"}
   [junit4]   2> 412145 T829 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 412146 T829 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 412146 T859 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 412146 T859 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 412147 T829 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 412147 T858 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 412148 T858 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 413146 T859 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 413147 T859 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 413147 T859 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 413148 T859 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 413148 T859 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 413148 T859 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/shard-1-001/cores/collection1/'
   [junit4]   2> 413163 T859 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 413165 T859 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 413174 T859 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 413193 T859 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 413232 T859 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 413334 T859 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 413343 T859 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 413739 T859 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 413761 T859 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 413764 T859 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 413778 T859 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 413783 T859 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 413788 T859 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 413789 T859 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 413789 T859 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 413790 T859 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 413790 T859 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 413791 T859 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 413791 T859 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 413791 T859 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 413792 T859 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 413792 T859 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@30f6bbb3
   [junit4]   2> 413793 T859 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/shard-1-001/cores/collection1/data
   [junit4]   2> 413794 T859 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 413794 T859 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 413795 T859 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 79C53BF88934A731-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 413795 T859 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=47, maxMergeAtOnceExplicit=39, maxMergedSegmentMB=91.076171875, floorSegmentMB=1.6279296875, forceMergeDeletesPctAllowed=16.892053792183308, segmentsPerTier=25.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 413796 T859 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7bd699b1 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@460472fb),segFN=segments_1,generation=1}
   [junit4]   2> 413796 T859 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 413806 T859 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 413807 T859 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 413807 T859 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 413807 T859 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 413808 T859 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 413808 T859 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 413809 T859 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 413809 T859 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 413809 T859 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 413838 T859 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 413841 T859 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 413844 T859 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 413847 T859 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 413857 T859 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/system,/admin/file,/admin/logging,/schema,standard,/update/json,/admin/properties,/admin/ping,/config,/update/csv,/admin/threads,/admin/segments,/update,/admin/plugins,/admin/luke,/update/json/docs,/admin/mbeans,/replication,/get
   [junit4]   2> 413859 T859 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 413861 T859 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 413862 T859 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 413864 T859 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 413864 T859 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 413865 T859 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=31, maxMergeAtOnceExplicit=38, maxMergedSegm

[...truncated too long message...]

.1:18478 18478
   [junit4]   2> 1364871 T4611 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		5	/solr/aliases.json
   [junit4]   2> 		5	/solr/clusterstate.json
   [junit4]   2> 		4	/solr/configs/conf1
   [junit4]   2> 		3	/solr/collections/repfacttest_c8n_1x3/state.json
   [junit4]   2> 		3	/solr/collections/collection1/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		5	/solr/live_nodes
   [junit4]   2> 		4	/solr/overseer/queue
   [junit4]   2> 		4	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1364872 T4610 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 7 connections to: http://127.0.0.1:49026/wi_f/c
   [junit4]   2> 1364872 T4610 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 2 connections to: http://127.0.0.1:18487/wi_f/c
   [junit4]   2> 1364872 T4610 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 9 connections to: http://127.0.0.1:18481/wi_f/c
   [junit4]   2> 1364873 T4610 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:15271/wi_f/c
   [junit4]   2> 1364873 T4610 C:control_collection S:shard1 c:collection1 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 1364873 T4610 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:49026/wi_f/c
   [junit4]   2> 1364874 T4610 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:18487/wi_f/c
   [junit4]   2> 1364874 T4610 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:18481/wi_f/c
   [junit4]   2> 1364874 T4610 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:15271/wi_f/c
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=79C53BF88934A731 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=pl -Dtests.timezone=Asia/Ujung_Pandang -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR    121s J0 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([79C53BF88934A731:F191042227C8CAC9]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 1364885 T4610 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 79C53BF88934A731-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {timestamp=Lucene50(blocksize=128), _version_=FSTOrd50, intDefault=FSTOrd50, id=PostingsFormat(name=Memory doPackFST= false), text=PostingsFormat(name=LuceneFixedGap), range_facet_l=Lucene50(blocksize=128), a_t=Lucene50(blocksize=128), multiDefault=Lucene50(blocksize=128)}, docValues:{timestamp=DocValuesFormat(name=Lucene50)}, sim=DefaultSimilarity, locale=pl, timezone=Asia/Ujung_Pandang
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=91286104,total=331350016
   [junit4]   2> NOTE: All tests run in this JVM: [UpdateRequestProcessorFactoryTest, BadIndexSchemaTest, BlockDirectoryTest, SolrRequestParserTest, TestShardHandlerFactory, TestDistribDocBasedVersion, RequiredFieldsTest, TestSchemaManager, TestSolrCoreProperties, AssignTest, FileUtilsTest, SimplePostToolTest, TestSortingResponseWriter, SegmentsInfoRequestHandlerTest, BadComponentTest, SuggesterTSTTest, HttpPartitionTest, TestSearchPerf, OverseerTest, TestIntervalFaceting, ScriptEngineTest, TestSort, CursorPagingTest, TestSolrConfigHandlerCloud, DistributedMLTComponentTest, ShowFileRequestHandlerTest, TestMergePolicyConfig, TestCSVResponseWriter, MergeStrategyTest, TestClassNameShortening, TestLazyCores, TestStressReorder, ZkNodePropsTest, TestBlendedInfixSuggestions, TestReRankQParserPlugin, ReplicationFactorTest]
   [junit4] Completed on J0 in 121.21s, 1 test, 1 error <<< FAILURES!

[...truncated 824 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 4 errors, 61 ignored (21 assumptions)

Total time: 54 minutes 27 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 227808967 bytes
Compression is 0.0%
Took 44 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2869 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2869/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:43024/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:43024/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([99AF15E1670C2C5:81CECE84B88CAF3D]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:58329/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:58329/collection1
	at __randomizedtesting.SeedInfo.seed([99AF15E1670C2C5:81CECE84B88CAF3D]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:58329/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:24922/repfacttest_c8n_1x3_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:24922/repfacttest_c8n_1x3_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([99AF15E1670C2C5:81CECE84B88CAF3D]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:24922/repfacttest_c8n_1x3_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9648 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/init-core-data-001
   [junit4]   2> 615000 T6342 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 615006 T6342 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 615007 T6343 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 615108 T6342 oasc.ZkTestServer.run start zk server on port:58321
   [junit4]   2> 615109 T6342 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 615110 T6342 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 615114 T6350 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2f6e72c8 name:ZooKeeperConnection Watcher:127.0.0.1:58321 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 615115 T6342 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 615115 T6342 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 615116 T6342 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 615119 T6342 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 615120 T6342 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 615122 T6353 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@41345000 name:ZooKeeperConnection Watcher:127.0.0.1:58321/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 615122 T6342 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 615123 T6342 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 615123 T6342 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 615125 T6342 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 615127 T6342 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 615129 T6342 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 615131 T6342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 615132 T6342 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 615135 T6342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 615136 T6342 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 615238 T6342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 615239 T6342 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 615242 T6342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 615242 T6342 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 615245 T6342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 615245 T6342 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 615248 T6342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 615248 T6342 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 615251 T6342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 615251 T6342 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 615253 T6342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 615254 T6342 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 615257 T6342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 615258 T6342 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 615261 T6342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 615262 T6342 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 615264 T6342 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 615265 T6342 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 617156 T6342 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/control-001/cores/collection1
   [junit4]   2> 617161 T6342 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 617169 T6342 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:58325
   [junit4]   2> 617170 T6342 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=58324, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/tempDir-001/control/data, hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/control-001/cores}
   [junit4]   2> 617171 T6342 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 617171 T6342 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/control-001/'
   [junit4]   2> 617231 T6342 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/control-001/solr.xml
   [junit4]   2> 617308 T6342 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/control-001/cores
   [junit4]   2> 617309 T6342 oasc.CoreContainer.<init> New CoreContainer 1220984615
   [junit4]   2> 617309 T6342 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/control-001/]
   [junit4]   2> 617309 T6342 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/control-001/lib
   [junit4]   2> 617310 T6342 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/control-001/lib).
   [junit4]   2> 617323 T6342 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 617324 T6342 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 617324 T6342 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 617325 T6342 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 617325 T6342 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 617325 T6342 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 617325 T6342 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 617326 T6342 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 617326 T6342 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 617326 T6342 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 617327 T6342 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 617327 T6342 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 617328 T6342 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 617328 T6342 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 617329 T6342 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 617329 T6342 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:58321/solr
   [junit4]   2> 617329 T6342 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 617330 T6342 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 617330 T6342 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 617332 T6365 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@cffa165 name:ZooKeeperConnection Watcher:127.0.0.1:58321 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 617333 T6342 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 617334 T6342 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 617336 T6342 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 617338 T6368 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2479fa96 name:ZooKeeperConnection Watcher:127.0.0.1:58321/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 617338 T6342 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 617340 T6342 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 617343 T6342 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 617345 T6342 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 617348 T6342 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 617350 T6342 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 617353 T6342 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 617355 T6342 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 617357 T6342 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 617358 T6342 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:58324_
   [junit4]   2> 617359 T6342 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:58324_
   [junit4]   2> 617361 T6342 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 617362 T6342 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 617364 T6342 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 617366 T6342 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:58324_
   [junit4]   2> 617367 T6342 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 617369 T6342 oasc.Overseer.start Overseer (id=93576390616481795-127.0.0.1:58324_-n_0000000000) starting
   [junit4]   2> 617371 T6342 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 617378 T6342 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 617379 T6370 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 617379 T6342 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 617380 T6369 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 617387 T6342 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/control-001/cores
   [junit4]   2> 617392 T6342 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, collection=control_collection, coreNodeName=, loadOnStartup=true, schema=schema.xml, dataDir=data/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/control-001/cores/collection1, transient=false, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/control-001/cores/collection1/, shard=, name=collection1}
   [junit4]   2> 617393 T6342 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/control-001/cores/collection1/
   [junit4]   2> 617393 T6342 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 617394 T6372 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 617394 T6372 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 617395 T6372 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 617395 T6368 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 617397 T6369 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58324",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:58324_",
   [junit4]   2> 	  "core":"collection1"} current state version: 0
   [junit4]   2> 617397 T6369 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58324",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:58324_",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 617398 T6369 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 617398 T6369 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 617399 T6368 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 618396 T6372 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 618397 T6372 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 618398 T6372 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 618398 T6372 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 618399 T6372 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 618399 T6372 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/control-001/cores/collection1/'
   [junit4]   2> 618440 T6372 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 618443 T6372 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 618455 T6372 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 618484 T6372 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 618543 T6372 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 618645 T6372 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 618666 T6372 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 619216 T6372 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 619233 T6372 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 619247 T6372 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 619260 T6372 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 619265 T6372 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 619270 T6372 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 619271 T6372 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 619272 T6372 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 619272 T6372 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 619273 T6372 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 619273 T6372 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 619273 T6372 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 619274 T6372 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 619274 T6372 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/control-001/cores/collection1/data/
   [junit4]   2> 619274 T6372 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7be98c44
   [junit4]   2> 619275 T6372 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/control-001/cores/collection1/data
   [junit4]   2> 619275 T6372 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/control-001/cores/collection1/data/index/
   [junit4]   2> 619276 T6372 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 619276 T6372 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/control-001/cores/collection1/data/index
   [junit4]   2> 619276 T6372 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=21, maxMergeAtOnceExplicit=45, maxMergedSegmentMB=22.4560546875, floorSegmentMB=2.12109375, forceMergeDeletesPctAllowed=0.7229221510722328, segmentsPerTier=36.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 619277 T6372 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4cc9997 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@14778af7),segFN=segments_1,generation=1}
   [junit4]   2> 619278 T6372 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 619283 T6372 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 619284 T6372 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 619284 T6372 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 619284 T6372 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 619285 T6372 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 619285 T6372 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 619285 T6372 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 619286 T6372 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 619286 T6372 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 619313 T6372 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 619316 T6372 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 619324 T6372 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 619331 T6372 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 619337 T6372 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /schema,/admin/properties,/update/json,standard,/update/csv,/admin/threads,/get,/admin/system,/config,/admin/segments,/admin/ping,/update,/replication,/admin/luke,/update/json/docs,/admin/plugins,/admin/mbeans,/admin/file,/admin/logging
   [junit4]   2> 619339 T6372 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 619341 T6372 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 619341 T6372 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 619342 T6372 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 619343 T6372 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 619343 T6372 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=36, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 619344 T6372 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4cc9997 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@14778af7),segFN=segments_1,generation=1}
   [junit4]   2> 619344 T6372 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 619345 T6372 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@550293df[collection1] main
   [junit4]   2> 619345 T6372 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 619346 T6372 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 619346 T6372 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 619347 T6372 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 619347 T6372 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 619347 T6372 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 619348 T6372 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 619348 T6372 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 619349 T6372 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 619349 T6372 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 619350 T6373 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@550293df[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 619350 T6372 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 619351 T6376 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:58324 collection:control_collection shard:shard1
   [junit4]   2> 619351 T6342 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 619352 T6342 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 619352 T6376 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 619356 T6342 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 619356 T6342 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 619358 T6376 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 619358 T6380 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@119d8512 name:ZooKeeperConnection Watcher:127.0.0.1:58321/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 619358 T6342 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 619359 T6342 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 619359 T6342 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 619360 T6368 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 619360 T6376 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 619361 T6376 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C647 name=collection1 org.apache.solr.core.SolrCore@30fd282 url=http://127.0.0.1:58324/collection1 node=127.0.0.1:58324_ C647_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:58324, state=down, node_name=127.0.0.1:58324_, core=collection1}
   [junit4]   2> 619361 T6376 C:control_collection S:shard1 C647 P58324 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:58324/collection1/
   [junit4]   2> 619361 T6376 C:control_collection S:shard1 C647 P58324 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 619362 T6376 C:control_collection S:shard1 C647 P58324 oasc.SyncStrategy.syncToMe http://127.0.0.1:58324/collection1/ has no replicas
   [junit4]   2> 619362 T6369 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 619362 T6376 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:58324/collection1/ shard1
   [junit4]   2> 619362 T6376 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 619364 T6368 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 619364 T6342 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 619364 T6380 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 619368 T6369 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58324",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 619371 T6368 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 619372 T6368 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 619372 T6380 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 619419 T6376 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:58324/collection1/ and leader is http://127.0.0.1:58324/collection1/
   [junit4]   2> 619420 T6376 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:58324
   [junit4]   2> 619420 T6376 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 619420 T6376 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 619420 T6376 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 619422 T6368 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 619423 T6369 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58324",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:58324_",
   [junit4]   2> 	  "core":"collection1"} current state version: 3
   [junit4]   2> 619424 T6369 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58324",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:58324_",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 619529 T6380 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 619529 T6368 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 621424 T6342 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/shard-1-001/cores/collection1
   [junit4]   2> 621428 T6342 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/shard-1-001
   [junit4]   2> 621429 T6342 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 621431 T6342 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:58330
   [junit4]   2> 621432 T6342 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, solrconfig=solrconfig.xml, hostPort=58329, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/tempDir-001/jetty1, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/shard-1-001/cores}
   [junit4]   2> 621432 T6342 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 621433 T6342 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/shard-1-001/'
   [junit4]   2> 621491 T6342 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/shard-1-001/solr.xml
   [junit4]   2> 621580 T6342 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/shard-1-001/cores
   [junit4]   2> 621581 T6342 oasc.CoreContainer.<init> New CoreContainer 1664212965
   [junit4]   2> 621581 T6342 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/shard-1-001/]
   [junit4]   2> 621581 T6342 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/shard-1-001/lib
   [junit4]   2> 621582 T6342 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/shard-1-001/lib).
   [junit4]   2> 621596 T6342 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 621596 T6342 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 621596 T6342 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 621597 T6342 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 621597 T6342 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 621597 T6342 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 621598 T6342 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 621598 T6342 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 621598 T6342 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 621599 T6342 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 621599 T6342 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 621600 T6342 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 621600 T6342 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 621600 T6342 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 621601 T6342 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 621601 T6342 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:58321/solr
   [junit4]   2> 621601 T6342 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 621602 T6342 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 621603 T6342 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 621604 T6392 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4d328568 name:ZooKeeperConnection Watcher:127.0.0.1:58321 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 621605 T6342 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 621606 T6342 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 621608 T6342 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 621609 T6395 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@514dbeef name:ZooKeeperConnection Watcher:127.0.0.1:58321/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 621609 T6342 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 621614 T6342 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 622619 T6342 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:58329_
   [junit4]   2> 622620 T6342 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:58329_
   [junit4]   2> 622622 T6342 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 622625 T6342 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/shard-1-001/cores
   [junit4]   2> 622627 T6342 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {schema=schema.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/shard-1-001/cores/collection1/, transient=false, loadOnStartup=true, dataDir=data/, collection=collection1, shard=, config=solrconfig.xml, coreNodeName=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/shard-1-001/cores/collection1, name=collection1}
   [junit4]   2> 622627 T6342 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/shard-1-001/cores/collection1/
   [junit4]   2> 622627 T6342 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 622628 T6396 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 622628 T6396 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 622629 T6368 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 622629 T6396 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 622630 T6369 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58329",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:58329_",
   [junit4]   2> 	  "core":"collection1"} current state version: 4
   [junit4]   2> 622630 T6369 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58329",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:58329_",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 622630 T6369 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 622631 T6369 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 622632 T6380 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 622632 T6368 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 622632 T6395 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 623630 T6396 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 623631 T6396 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 623631 T6396 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 623632 T6396 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 623632 T6396 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 623632 T6396 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/shard-1-001/cores/collection1/'
   [junit4]   2> 623655 T6396 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 623658 T6396 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 623666 T6396 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 623696 T6396 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 623806 T6396 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 623908 T6396 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 623915 T6396 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 624445 T6396 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 624456 T6396 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 624458 T6396 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 624476 T6396 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 624483 T6396 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 624487 T6396 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 624488 T6396 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 624488 T6396 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 624488 T6396 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 624489 T6396 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 624489 T6396 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 624489 T6396 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 624490 T6396 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 624490 T6396 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 624490 T6396 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7be98c44
   [junit4]   2> 624491 T6396 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/shard-1-001/cores/collection1/data
   [junit4]   2> 624492 T6396 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 624492 T6396 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 624492 T6396 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 99AF15E1670C2C5-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 624493 T6396 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=21, maxMergeAtOnceExplicit=45, maxMergedSegmentMB=22.4560546875, floorSegmentMB=2.12109375, forceMergeDeletesPctAllowed=0.7229221510722328, segmentsPerTier=36.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 624494 T6396 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2f278e8c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4dffa183),segFN=segments_1,generation=1}
   [junit4]   2> 624494 T6396 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 624501 T6396 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 624501 T6396 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 624502 T6396 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 624502 T6396 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 624502 T6396 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 624502 T6396 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 624503 T6396 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 624503 T6396 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 624503 T6396 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 624525 T6396 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 624528 T6396 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 624531 T6396 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 624533 T6396 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 624541 T6396 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /schema,/admin/properties,/update/json,standard,/update/csv,/admin/threads,/get,/admin/system,/config,/admin/segments,/admin/ping,/update,/replication,/admin/luke,/update/json/docs,/admin/plugins,/admin/mbeans,/admin/file,/admin/logging
   [junit4]   2> 624543 T6396 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 624544 T6396 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 624545 T6396 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 624546 T6396 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 624546 T6396 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 624547 T6396 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=36, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 624548 T6396 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2f278e8c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4dffa183),segFN=segments_1,generation=1}
   [junit4]   2> 624548 T6396 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 624549 T6396 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@29d465bf[collection1] main
   [junit4]   2> 624549 T6396 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 624550 T6396 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 624550 T6396 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 624551 T6396 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 624551 T6396 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 624551 T6396 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 624552 T6396 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 624552 T6396 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 624552 T6396 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 624553 T6396 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 624554 T6397 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@29d465bf[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 624554 T6396 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 624555 T6400 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:58329 collection:collection1 shard:shard1
   [junit4]   2> 624555 T6342 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 624556 T6342 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 624556 T6400 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 624560 T6400 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 624561 T6368 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 624562 T6400 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 624562 T6400 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C648 name=collection1 org.apache.solr.core.SolrCore@2909e46f url=http://127.0.0.1:58329/collection1 node=127.0.0.1:58329_ C648_STATE=coll:collection1 core:collection1 props:{base_url=http://127.0.0.1:58329, state=down, node_name=127.0.0.1:58329_, core=collection1}
   [junit4]   2> 624562 T6400 C:collection1 S:shard1 C648 P58329 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:58329/collection1/
   [junit4]   2> 624562 T6369 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 624562 T6400 C:collection1 S:shard1 C648 P58329 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 624563 T6400 C:collection1 S:shard1 C648 P58329 oasc.SyncStrategy.syncToMe http://127.0.0.1:58329/collection1/ has no replicas
   [junit4]   2> 624563 T6400 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:58329/collection1/ shard1
   [junit4]   2> 624563 T6400 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 624564 T6368 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 624564 T6380 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 624564 T6395 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 624568 T6368 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 624568 T6369 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58329",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 624672 T6368 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 624672 T6380 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 624672 T6395 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 624720 T6400 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:58329/collection1/ and leader is http://127.0.0.1:58329/collection1/
   [junit4]   2> 624721 T6400 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:58329
   [junit4]   2> 624721 T6400 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 624721 T6400 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 624721 T6400 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 624723 T6368 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 624724 T6369 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58329",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:58329_",
   [junit4]   2> 	  "core":"collection1"} current state version: 7
   [junit4]   2> 624725 T6369 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58329",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:58329_",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 624828 T6380 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 624828 T6368 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nod

[...truncated too long message...]

d o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> 1671710 T8381 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:42981 42981
   [junit4]   2> 1672821 T8382 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:42981 42981
   [junit4]   2> 1672823 T8382 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		6	/solr/aliases.json
   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 		3	/solr/collections/c8n_1x3_commits/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 		4	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1672823 T8381 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 15 connections to: http://127.0.0.1:43024/
   [junit4]   2> 1672824 T8381 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:43121/
   [junit4]   2> 1672824 T8381 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:43002/
   [junit4]   2> 1672824 T8381 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 3 connections to: http://127.0.0.1:43038/
   [junit4]   2> 1672824 T8381 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:43157/
   [junit4]   2> 1672825 T8381 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:43024/
   [junit4]   2> 1672825 T8381 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:43121/
   [junit4]   2> 1672826 T8381 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:43002/
   [junit4]   2> 1672826 T8381 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:43038/
   [junit4]   2> 1672826 T8381 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:43157/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderInitiatedRecoveryOnCommitTest -Dtests.method=test -Dtests.seed=99AF15E1670C2C5 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=lv_LV -Dtests.timezone=America/Cancun -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   83.0s J0 | LeaderInitiatedRecoveryOnCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:43024/c8n_1x3_commits_shard1_replica3
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([99AF15E1670C2C5:81CECE84B88CAF3D]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 48 more
   [junit4]   2> 1672849 T8381 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 99AF15E1670C2C5-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=false,coord=crazy): {}, locale=lv_LV, timezone=America/Cancun
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=173463880,total=297271296
   [junit4]   2> NOTE: All tests run in this JVM: [TestReversedWildcardFilterFactory, PolyFieldTest, CopyFieldTest, DistribDocExpirationUpdateProcessorTest, TestUpdate, TestMergePolicyConfig, TestManagedSynonymFilterFactory, TestLeaderElectionZkExpiry, TestConfigOverlay, SuggesterTSTTest, TestManagedStopFilterFactory, BadIndexSchemaTest, PeerSyncTest, TestSolrIndexConfig, ClusterStateTest, SignatureUpdateProcessorFactoryTest, TestCursorMarkWithoutUniqueKey, TestPerFieldSimilarity, SolrIndexSplitterTest, TestRecoveryHdfs, TestOrdValues, DistributedFacetPivotWhiteBoxTest, TestExactStatsCache, TestAddFieldRealTimeGet, NoCacheHeaderTest, TestLFUCache, CacheHeaderTest, OpenCloseCoreStressTest, TestChildDocTransformer, TestReloadAndDeleteDocs, RAMDirectoryFactoryTest, QueryElevationComponentTest, URLClassifyProcessorTest, TestFieldResource, OverseerStatusTest, TestSolrConfigHandlerCloud, RecoveryAfterSoftCommitTest, FileUtilsTest, CursorMarkTest, DistributedSuggestComponentTest, RecoveryZkTest, TestInitQParser, MultiTermTest, TestMiniSolrCloudClusterSSL, TestRTGBase, TestExactSharedStatsCache, DistributedQueryComponentCustomSortTest, OverseerTest, LeaderElectionIntegrationTest, TestIntervalFaceting, TestAnalyzedSuggestions, TestInitParams, TestMiniSolrCloudCluster, DocExpirationUpdateProcessorFactoryTest, SolrPluginUtilsTest, RequestHandlersTest, SyncSliceTest, TestDistributedSearch, LeaderInitiatedRecoveryOnCommitTest]
   [junit4] Completed on J0 in 83.03s, 1 test, 1 error <<< FAILURES!

[...truncated 718 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 57 minutes 41 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 225029892 bytes
Compression is 0.0%
Took 46 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2868 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2868/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:48214/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48214/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([22900A47AEDD8F59:AAC4359D0021E2A1]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:29548/f_mm/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:29548/f_mm/collection1
	at __randomizedtesting.SeedInfo.seed([22900A47AEDD8F59:AAC4359D0021E2A1]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:29548/f_mm/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:23476/repfacttest_c8n_1x3_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:23476/repfacttest_c8n_1x3_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([22900A47AEDD8F59:AAC4359D0021E2A1]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:23476/repfacttest_c8n_1x3_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9293 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/init-core-data-001
   [junit4]   2> 94700 T177 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 94710 T177 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 94711 T178 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 94811 T177 oasc.ZkTestServer.run start zk server on port:48181
   [junit4]   2> 94812 T177 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 94821 T177 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 94824 T185 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6edd2f8c name:ZooKeeperConnection Watcher:127.0.0.1:48181 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 94824 T177 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 94825 T177 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 94825 T177 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 94839 T177 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 94840 T177 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 94842 T188 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@10afa1ee name:ZooKeeperConnection Watcher:127.0.0.1:48181/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 94842 T177 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 94843 T177 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 94843 T177 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 94846 T177 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 94848 T177 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 94850 T177 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 94852 T177 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 94853 T177 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 94857 T177 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 94858 T177 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 94861 T177 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 94862 T177 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 94864 T177 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 94865 T177 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 94868 T177 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 94868 T177 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 94871 T177 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 94871 T177 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 94874 T177 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 94874 T177 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 94877 T177 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 94877 T177 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 94880 T177 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 94881 T177 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 94883 T177 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 94884 T177 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 94886 T177 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 94887 T177 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 96544 T177 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/control-001/cores/collection1
   [junit4]   2> 96548 T177 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 96553 T177 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:48188
   [junit4]   2> 96554 T177 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=48187, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/control-001/cores}
   [junit4]   2> 96555 T177 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 96555 T177 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/control-001/'
   [junit4]   2> 96610 T177 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/control-001/solr.xml
   [junit4]   2> 96665 T177 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/control-001/cores
   [junit4]   2> 96666 T177 oasc.CoreContainer.<init> New CoreContainer 390184556
   [junit4]   2> 96667 T177 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/control-001/]
   [junit4]   2> 96667 T177 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/control-001/lib
   [junit4]   2> 96667 T177 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/control-001/lib).
   [junit4]   2> 96678 T177 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 96679 T177 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 96679 T177 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 96679 T177 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 96679 T177 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 96680 T177 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 96680 T177 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 96680 T177 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 96680 T177 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 96681 T177 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 96681 T177 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 96682 T177 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 96682 T177 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 96682 T177 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 96683 T177 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 96683 T177 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:48181/solr
   [junit4]   2> 96683 T177 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 96684 T177 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 96685 T177 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 96688 T200 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@460e247a name:ZooKeeperConnection Watcher:127.0.0.1:48181 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 96688 T177 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 96689 T177 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 96692 T177 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 96694 T203 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@818d18d name:ZooKeeperConnection Watcher:127.0.0.1:48181/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 96694 T177 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 96697 T177 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 96701 T177 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 96705 T177 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 96709 T177 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 96712 T177 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 96716 T177 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 96719 T177 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 96722 T177 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 96724 T177 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48187_
   [junit4]   2> 96725 T177 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:48187_
   [junit4]   2> 96728 T177 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 96731 T177 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 96734 T177 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 96737 T177 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:48187_
   [junit4]   2> 96737 T177 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 96740 T177 oasc.Overseer.start Overseer (id=93575401889988611-127.0.0.1:48187_-n_0000000000) starting
   [junit4]   2> 96743 T177 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 96757 T177 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 96758 T205 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 96758 T177 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 96759 T204 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 96767 T177 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/control-001/cores
   [junit4]   2> 96772 T177 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=control_collection, schema=schema.xml, loadOnStartup=true, dataDir=data/, config=solrconfig.xml, name=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/control-001/cores/collection1/, coreNodeName=, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/control-001/cores/collection1, shard=}
   [junit4]   2> 96773 T177 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/control-001/cores/collection1/
   [junit4]   2> 96773 T177 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 96775 T207 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 96776 T207 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 96778 T203 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 96778 T207 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 96780 T204 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48187",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:48187_",
   [junit4]   2> 	  "roles":null} current state version: 0
   [junit4]   2> 96781 T204 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48187",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:48187_",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 96781 T204 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 96781 T204 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 96784 T203 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 97779 T207 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 97780 T207 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 97782 T207 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 97782 T207 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 97784 T207 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 97784 T207 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/control-001/cores/collection1/'
   [junit4]   2> 97820 T207 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 97826 T207 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 97854 T207 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 97899 T207 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 98019 T207 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 98021 T207 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 98071 T207 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 98989 T207 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 99051 T207 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 99058 T207 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 99130 T207 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 99136 T207 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 99143 T207 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 99151 T207 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 99152 T207 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 99152 T207 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 99154 T207 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 99154 T207 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 99171 T207 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 99171 T207 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 99172 T207 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/control-001/cores/collection1/data/
   [junit4]   2> 99172 T207 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2101f0d8
   [junit4]   2> 99173 T207 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/control-001/cores/collection1/data
   [junit4]   2> 99174 T207 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/control-001/cores/collection1/data/index/
   [junit4]   2> 99174 T207 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 99175 T207 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/control-001/cores/collection1/data/index
   [junit4]   2> 99176 T207 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=46, maxMergedSegmentMB=24.0087890625, floorSegmentMB=0.65625, forceMergeDeletesPctAllowed=21.263751956518085, segmentsPerTier=11.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 99177 T207 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2f4ea129 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5336a302),segFN=segments_1,generation=1}
   [junit4]   2> 99177 T207 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 99186 T207 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 99186 T207 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 99187 T207 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 99187 T207 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 99187 T207 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 99188 T207 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 99188 T207 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 99189 T207 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 99189 T207 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 99212 T207 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 99215 T207 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 99219 T207 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 99223 T207 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 99233 T207 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/threads,/update,/admin/file,/admin/properties,/admin/mbeans,standard,/config,/admin/plugins,/schema,/replication,/update/json,/admin/luke,/update/csv,/admin/segments,/admin/logging,/admin/system,/get,/update/json/docs,/admin/ping
   [junit4]   2> 99236 T207 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 99238 T207 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 99238 T207 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 99240 T207 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 99240 T207 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 99241 T207 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=47, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 99242 T207 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2f4ea129 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5336a302),segFN=segments_1,generation=1}
   [junit4]   2> 99243 T207 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 99243 T207 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@75efd6b1[collection1] main
   [junit4]   2> 99244 T207 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 99246 T207 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 99246 T207 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 99247 T207 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 99248 T207 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 99248 T207 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 99249 T207 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 99250 T207 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 99250 T207 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 99251 T207 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 99252 T208 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@75efd6b1[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 99253 T207 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 99254 T211 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:48187 collection:control_collection shard:shard1
   [junit4]   2> 99255 T177 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 99256 T177 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 99256 T211 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 99261 T177 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 99262 T177 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 99264 T215 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4eccd59b name:ZooKeeperConnection Watcher:127.0.0.1:48181/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 99264 T177 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 99265 T177 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 99265 T177 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 99266 T211 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 99269 T203 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 99270 T211 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 99270 T211 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C22 name=collection1 org.apache.solr.core.SolrCore@4e4c86aa url=http://127.0.0.1:48187/collection1 node=127.0.0.1:48187_ C22_STATE=coll:control_collection core:collection1 props:{core=collection1, state=down, base_url=http://127.0.0.1:48187, node_name=127.0.0.1:48187_}
   [junit4]   2> 99270 T211 C:control_collection S:shard1 C22 P48187 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:48187/collection1/
   [junit4]   2> 99271 T204 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 99271 T211 C:control_collection S:shard1 C22 P48187 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 99271 T177 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 99272 T211 C:control_collection S:shard1 C22 P48187 oasc.SyncStrategy.syncToMe http://127.0.0.1:48187/collection1/ has no replicas
   [junit4]   2> 99273 T211 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:48187/collection1/ shard1
   [junit4]   2> 99273 T211 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 99274 T203 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 99274 T215 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 99283 T203 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 99284 T204 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48187",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 99390 T203 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 99390 T215 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 99436 T211 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:48187/collection1/ and leader is http://127.0.0.1:48187/collection1/
   [junit4]   2> 99437 T211 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:48187
   [junit4]   2> 99437 T211 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 99437 T211 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 99438 T211 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 99440 T203 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 99442 T204 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48187",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:48187_",
   [junit4]   2> 	  "roles":null} current state version: 3
   [junit4]   2> 99443 T204 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48187",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:48187_",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 99549 T203 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 99549 T215 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 100680 T177 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/shard-1-001/cores/collection1
   [junit4]   2> 100683 T177 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/shard-1-001
   [junit4]   2> 100684 T177 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 100687 T177 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:48206
   [junit4]   2> 100688 T177 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/shard-1-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/tempDir-001/jetty1, hostPort=48205, solrconfig=solrconfig.xml, hostContext=/}
   [junit4]   2> 100688 T177 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 100689 T177 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/shard-1-001/'
   [junit4]   2> 100738 T177 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/shard-1-001/solr.xml
   [junit4]   2> 100799 T177 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/shard-1-001/cores
   [junit4]   2> 100800 T177 oasc.CoreContainer.<init> New CoreContainer 1893792307
   [junit4]   2> 100801 T177 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/shard-1-001/]
   [junit4]   2> 100801 T177 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/shard-1-001/lib
   [junit4]   2> 100802 T177 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/shard-1-001/lib).
   [junit4]   2> 100817 T177 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 100818 T177 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 100818 T177 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 100818 T177 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 100819 T177 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 100819 T177 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 100819 T177 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 100820 T177 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 100820 T177 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 100821 T177 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 100821 T177 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 100822 T177 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 100822 T177 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 100823 T177 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 100823 T177 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 100824 T177 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:48181/solr
   [junit4]   2> 100824 T177 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 100824 T177 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 100825 T177 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 100828 T227 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5a6d893 name:ZooKeeperConnection Watcher:127.0.0.1:48181 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 100828 T177 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 100829 T177 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 100832 T177 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 100833 T230 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@286bb11a name:ZooKeeperConnection Watcher:127.0.0.1:48181/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 100834 T177 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 100840 T177 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 101846 T177 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48205_
   [junit4]   2> 101847 T177 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:48205_
   [junit4]   2> 101850 T177 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 101853 T177 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/shard-1-001/cores
   [junit4]   2> 101855 T177 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {shard=, coreNodeName=, loadOnStartup=true, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/shard-1-001/cores/collection1, config=solrconfig.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/shard-1-001/cores/collection1/, schema=schema.xml, collection=collection1, transient=false, dataDir=data/, name=collection1}
   [junit4]   2> 101855 T177 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/shard-1-001/cores/collection1/
   [junit4]   2> 101856 T177 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 101857 T231 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 101857 T231 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 101858 T203 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 101858 T231 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 101859 T204 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48205",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:48205_",
   [junit4]   2> 	  "roles":null} current state version: 4
   [junit4]   2> 101859 T204 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48205",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:48205_",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 101860 T204 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 101860 T204 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 101861 T203 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 101861 T230 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 101861 T215 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 102858 T231 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 102859 T231 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 102860 T231 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 102860 T231 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 102861 T231 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 102861 T231 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/shard-1-001/cores/collection1/'
   [junit4]   2> 102876 T231 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 102879 T231 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 102892 T231 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 102916 T231 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 102993 T231 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 102996 T231 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 103029 T231 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 103423 T231 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 103439 T231 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 103441 T231 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 103460 T231 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 103465 T231 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 103468 T231 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 103469 T231 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 103470 T231 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 103470 T231 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 103471 T231 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 103471 T231 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 103471 T231 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 103471 T231 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 103472 T231 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 103472 T231 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2101f0d8
   [junit4]   2> 103473 T231 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/shard-1-001/cores/collection1/data
   [junit4]   2> 103473 T231 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 103473 T231 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 103474 T231 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 22900A47AEDD8F59-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 103474 T231 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=46, maxMergedSegmentMB=24.0087890625, floorSegmentMB=0.65625, forceMergeDeletesPctAllowed=21.263751956518085, segmentsPerTier=11.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 103475 T231 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2bb1d1fa lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@27508a7f),segFN=segments_1,generation=1}
   [junit4]   2> 103475 T231 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 103483 T231 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 103483 T231 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 103483 T231 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 103483 T231 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 103484 T231 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 103484 T231 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 103484 T231 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 103485 T231 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 103485 T231 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 103498 T231 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 103500 T231 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 103502 T231 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 103503 T231 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 103511 T231 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/threads,/update,/admin/file,/admin/properties,/admin/mbeans,standard,/config,/admin/plugins,/schema,/replication,/update/json,/admin/luke,/update/csv,/admin/segments,/admin/logging,/admin/system,/get,/update/json/docs,/admin/ping
   [junit4]   2> 103514 T231 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 103515 T231 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 103515 T231 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 103516 T231 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 103516 T231 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 103518 T231 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=47, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 103518 T231 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2bb1d1fa lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@27508a7f),segFN=segments_1,generation=1}
   [junit4]   2> 103518 T231 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 103519 T231 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@2942140a[collection1] main
   [junit4]   2> 103519 T231 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 103520 T231 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 103521 T231 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 103521 T231 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 103521 T231 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 103522 T231 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 103522 T231 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 103522 T231 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 103523 T231 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 103523 T231 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 103524 T232 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2942140a[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 103524 T231 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 103525 T235 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:48205 collection:collection1 shard:shard1
   [junit4]   2> 103526 T177 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 103526 T177 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 103527 T235 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 103535 T235 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 103538 T203 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 103538 T235 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 103539 T235 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C23 name=collection1 org.apache.solr.core.SolrCore@18798e38 url=http://127.0.0.1:48205/collection1 node=127.0.0.1:48205_ C23_STATE=coll:collection1 core:collection1 props:{core=collection1, state=down, base_url=http://127.0.0.1:48205, node_name=127.0.0.1:48205_}
   [junit4]   2> 103539 T235 C:collection1 S:shard1 C23 P48205 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:48205/collection1/
   [junit4]   2> 103539 T204 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 103539 T235 C:collection1 S:shard1 C23 P48205 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 103540 T235 C:collection1 S:shard1 C23 P48205 oasc.SyncStrategy.syncToMe http://127.0.0.1:48205/collection1/ has no replicas
   [junit4]   2> 103540 T235 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:48205/collection1/ shard1
   [junit4]   2> 103541 T235 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 103542 T203 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 103542 T230 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 103542 T215 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 103549 T203 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 103550 T204 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48205",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 103656 T215 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 103656 T230 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 103656 T203 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 103702 T235 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:48205/collection1/ and leader is http://127.0.0.1:48205/collection1/
   [junit4]   2> 103703 T235 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:48205
   [junit4]   2> 103703 T235 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 103703 T235 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 103704 T235 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 103706 T203 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 103707 T204 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48205",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:48205_",
   [junit4]   2> 	  "roles":null} current state version: 7
   [junit4]   2> 103708 T204 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48205",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:48205_",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 103813 T215 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 103813 T203 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 103813 T230 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nod

[...truncated too long message...]

lection-queue-work
   [junit4]   2> 		4	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1056546 T1703 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:23443/
   [junit4]   2> 1056546 T1703 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 10 connections to: http://127.0.0.1:23476/
   [junit4]   2> 1056546 T1703 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:23451/
   [junit4]   2> 1056546 T1703 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:23459/
   [junit4]   2> 1056547 T1703 C:control_collection S:shard1 c:collection1 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 1056547 T1703 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:23443/
   [junit4]   2> 1056547 T1703 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:23476/
   [junit4]   2> 1056547 T1703 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:23451/
   [junit4]   2> 1056548 T1703 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:23459/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=22900A47AEDD8F59 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=sr_ME_#Latn -Dtests.timezone=Africa/Casablanca -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   73.5s J1 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:23476/repfacttest_c8n_1x3_shard1_replica2
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([22900A47AEDD8F59:AAC4359D0021E2A1]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:23476/repfacttest_c8n_1x3_shard1_replica2
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 48 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 53 more
   [junit4]   2> 1056569 T1703 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 22900A47AEDD8F59-001
   [junit4]   2> 73531 T1702 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {timestamp=Lucene50(blocksize=128), _version_=Lucene50(blocksize=128), intDefault=Lucene50(blocksize=128), multiDefault=PostingsFormat(name=Memory doPackFST= false), a_t=Lucene50(blocksize=128), text=FST50, range_facet_l=Lucene50(blocksize=128), id=PostingsFormat(name=Asserting)}, docValues:{timestamp=DocValuesFormat(name=Lucene50)}, sim=DefaultSimilarity, locale=sr_ME_#Latn, timezone=Africa/Casablanca
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=138359680,total=307757056
   [junit4]   2> NOTE: All tests run in this JVM: [TestExceedMaxTermLength, SSLMigrationTest, TestBulkSchemaConcurrent, TestTrackingShardHandlerFactory, TestSolrQueryParser, SolrRequestParserTest, TestCopyFieldCollectionResource, TestDistributedSearch, SharedFSAutoReplicaFailoverTest, TestArbitraryIndexDir, TestWordDelimiterFilterFactory, TestSortingResponseWriter, TestSearcherReuse, PreAnalyzedUpdateProcessorTest, TestJsonRequest, TestPHPSerializedResponseWriter, QueryParsingTest, ScriptEngineTest, TestSolrConfigHandler, TestUtils, DOMUtilTest, TestSimpleQParserPlugin, QueryElevationComponentTest, DistributedQueryElevationComponentTest, TestManagedSchema, TestManagedSynonymFilterFactory, TestBlendedInfixSuggestions, ZkControllerTest, LeaderFailoverAfterPartitionTest, DistributedFacetPivotSmallTest, CloudExitableDirectoryReaderTest, TestOmitPositions, TestNoOpRegenerator, TestReplicationHandlerBackup, TestSolrDeletionPolicy2, TestJsonFacets, TestXIncludeConfig, TestAnalyzedSuggestions, TestBinaryResponseWriter, PrimitiveFieldTypeTest, FieldAnalysisRequestHandlerTest, SuggestComponentTest, ReplicationFactorTest]
   [junit4] Completed on J1 in 74.32s, 1 test, 1 error <<< FAILURES!

[...truncated 889 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 53 minutes 1 second
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 212316423 bytes
Compression is 0.0%
Took 41 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2867 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2867/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:10211/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:10211/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([4B62220C8632FEF:8CE21DFA669F4217]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:51971/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:51971/collection1
	at __randomizedtesting.SeedInfo.seed([4B62220C8632FEF:8CE21DFA669F4217]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:51971/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:10203/io/q/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:10203/io/q/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([4B62220C8632FEF:8CE21DFA669F4217]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:10203/io/q/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9804 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/init-core-data-001
   [junit4]   2> 649119 T1789 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 649127 T1789 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 649128 T1790 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 649228 T1789 oasc.ZkTestServer.run start zk server on port:51949
   [junit4]   2> 649229 T1789 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 649230 T1789 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 649236 T1797 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7ad73a2f name:ZooKeeperConnection Watcher:127.0.0.1:51949 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 649236 T1789 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 649237 T1789 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 649237 T1789 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 649241 T1789 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 649241 T1789 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 649243 T1800 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5705db19 name:ZooKeeperConnection Watcher:127.0.0.1:51949/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 649243 T1789 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 649243 T1789 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 649244 T1789 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 649246 T1789 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 649247 T1789 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 649248 T1789 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 649250 T1789 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 649250 T1789 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 649253 T1789 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 649253 T1789 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 649355 T1789 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 649356 T1789 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 649357 T1789 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 649357 T1789 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 649359 T1789 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 649359 T1789 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 649361 T1789 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 649361 T1789 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 649363 T1789 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 649363 T1789 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 649364 T1789 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 649365 T1789 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 649366 T1789 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 649367 T1789 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 649368 T1789 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 649369 T1789 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 649370 T1789 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 649370 T1789 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 650838 T1789 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/control-001/cores/collection1
   [junit4]   2> 650842 T1789 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 650845 T1789 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:51957
   [junit4]   2> 650845 T1789 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/tempDir-001/control/data, hostPort=51956}
   [junit4]   2> 650846 T1789 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 650846 T1789 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/control-001/'
   [junit4]   2> 650891 T1789 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/control-001/solr.xml
   [junit4]   2> 650947 T1789 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/control-001/cores
   [junit4]   2> 650948 T1789 oasc.CoreContainer.<init> New CoreContainer 1546452661
   [junit4]   2> 650948 T1789 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/control-001/]
   [junit4]   2> 650948 T1789 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/control-001/lib
   [junit4]   2> 650949 T1789 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/control-001/lib).
   [junit4]   2> 650958 T1789 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 650958 T1789 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 650959 T1789 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 650959 T1789 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 650959 T1789 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 650959 T1789 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 650959 T1789 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 650960 T1789 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 650960 T1789 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 650960 T1789 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 650960 T1789 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 650961 T1789 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 650961 T1789 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 650961 T1789 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 650961 T1789 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 650962 T1789 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51949/solr
   [junit4]   2> 650962 T1789 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 650962 T1789 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 650963 T1789 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 650964 T1812 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4b3b26e2 name:ZooKeeperConnection Watcher:127.0.0.1:51949 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 650964 T1789 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 650965 T1789 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 650967 T1789 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 650968 T1815 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@674694e4 name:ZooKeeperConnection Watcher:127.0.0.1:51949/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 650969 T1789 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 650970 T1789 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 650972 T1789 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 650973 T1789 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 650974 T1789 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 650976 T1789 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 650977 T1789 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 650978 T1789 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 650979 T1789 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 650980 T1789 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51956_
   [junit4]   2> 650980 T1789 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:51956_
   [junit4]   2> 650981 T1789 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 650982 T1789 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 650983 T1789 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 650984 T1789 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:51956_
   [junit4]   2> 650984 T1789 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 650985 T1789 oasc.Overseer.start Overseer (id=93574595015606275-127.0.0.1:51956_-n_0000000000) starting
   [junit4]   2> 650987 T1789 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 650991 T1789 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 650991 T1817 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 650991 T1816 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 650991 T1789 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 650994 T1789 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/control-001/cores
   [junit4]   2> 650996 T1789 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=control_collection, loadOnStartup=true, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/control-001/cores/collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/control-001/cores/collection1/, shard=, schema=schema.xml, name=collection1, coreNodeName=, config=solrconfig.xml, dataDir=data/}
   [junit4]   2> 650997 T1789 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/control-001/cores/collection1/
   [junit4]   2> 650997 T1789 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 650998 T1819 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 650998 T1819 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 650998 T1819 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 650998 T1815 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 650999 T1816 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:51956_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51956",
   [junit4]   2> 	  "state":"down"} current state version: 0
   [junit4]   2> 650999 T1816 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:51956_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51956",
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 651000 T1816 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 651000 T1816 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 651001 T1815 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 651999 T1819 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 652000 T1819 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 652000 T1819 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 652000 T1819 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 652001 T1819 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 652001 T1819 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/control-001/cores/collection1/'
   [junit4]   2> 652012 T1819 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 652014 T1819 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 652021 T1819 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 652035 T1819 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 652067 T1819 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 652168 T1819 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 652175 T1819 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 652445 T1819 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 652455 T1819 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 652457 T1819 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 652466 T1819 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 652469 T1819 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 652471 T1819 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 652472 T1819 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 652472 T1819 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 652472 T1819 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 652473 T1819 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 652473 T1819 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 652473 T1819 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 652473 T1819 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 652473 T1819 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/control-001/cores/collection1/data/
   [junit4]   2> 652474 T1819 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@347fd24a
   [junit4]   2> 652474 T1819 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/control-001/cores/collection1/data
   [junit4]   2> 652475 T1819 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/control-001/cores/collection1/data/index/
   [junit4]   2> 652475 T1819 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 652475 T1819 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/control-001/cores/collection1/data/index
   [junit4]   2> 652475 T1819 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=49, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 652476 T1819 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3a635c69 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@51480f71),segFN=segments_1,generation=1}
   [junit4]   2> 652476 T1819 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 652480 T1819 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 652480 T1819 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 652480 T1819 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 652481 T1819 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 652481 T1819 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 652481 T1819 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 652481 T1819 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 652482 T1819 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 652482 T1819 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 652494 T1819 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 652495 T1819 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 652497 T1819 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 652498 T1819 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 652502 T1819 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /get,/admin/logging,/schema,/update/json/docs,/update/json,/admin/luke,/admin/file,/update,/admin/threads,/admin/segments,standard,/admin/plugins,/admin/mbeans,/update/csv,/config,/admin/system,/replication,/admin/ping,/admin/properties
   [junit4]   2> 652504 T1819 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 652505 T1819 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 652505 T1819 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 652506 T1819 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 652506 T1819 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 652507 T1819 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=11, maxMergeAtOnceExplicit=10, maxMergedSegmentMB=38.0224609375, floorSegmentMB=1.4755859375, forceMergeDeletesPctAllowed=27.85073561451421, segmentsPerTier=25.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7668826767996076
   [junit4]   2> 652507 T1819 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3a635c69 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@51480f71),segFN=segments_1,generation=1}
   [junit4]   2> 652507 T1819 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 652508 T1819 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@c1960e8[collection1] main
   [junit4]   2> 652508 T1819 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 652509 T1819 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 652509 T1819 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 652509 T1819 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 652509 T1819 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 652509 T1819 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 652510 T1819 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 652510 T1819 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 652510 T1819 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 652510 T1819 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 652511 T1819 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 652511 T1820 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@c1960e8[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 652511 T1823 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:51956 collection:control_collection shard:shard1
   [junit4]   2> 652512 T1789 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 652512 T1789 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 652512 T1823 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 652514 T1789 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 652515 T1789 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 652516 T1827 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2891265b name:ZooKeeperConnection Watcher:127.0.0.1:51949/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 652517 T1789 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 652517 T1789 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 652517 T1823 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 652517 T1789 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 652519 T1815 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 652519 T1823 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 652520 T1823 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C251 name=collection1 org.apache.solr.core.SolrCore@302c9cb6 url=http://127.0.0.1:51956/collection1 node=127.0.0.1:51956_ C251_STATE=coll:control_collection core:collection1 props:{core=collection1, node_name=127.0.0.1:51956_, base_url=http://127.0.0.1:51956, state=down}
   [junit4]   2> 652520 T1823 C:control_collection S:shard1 C251 P51956 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:51956/collection1/
   [junit4]   2> 652520 T1823 C:control_collection S:shard1 C251 P51956 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 652520 T1816 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 652520 T1823 C:control_collection S:shard1 C251 P51956 oasc.SyncStrategy.syncToMe http://127.0.0.1:51956/collection1/ has no replicas
   [junit4]   2> 652520 T1789 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 652520 T1823 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:51956/collection1/ shard1
   [junit4]   2> 652521 T1823 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 652521 T1815 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 652521 T1827 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 652525 T1815 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 652525 T1816 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51956",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 652628 T1815 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 652628 T1827 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 652678 T1823 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:51956/collection1/ and leader is http://127.0.0.1:51956/collection1/
   [junit4]   2> 652679 T1823 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:51956
   [junit4]   2> 652679 T1823 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 652679 T1823 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 652679 T1823 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 652680 T1815 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 652681 T1816 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:51956_",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51956",
   [junit4]   2> 	  "state":"active"} current state version: 3
   [junit4]   2> 652682 T1816 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:51956_",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51956",
   [junit4]   2> 	  "state":"active"}
   [junit4]   2> 652785 T1815 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 652785 T1827 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 653491 T1789 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/shard-1-001/cores/collection1
   [junit4]   2> 653493 T1789 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/shard-1-001
   [junit4]   2> 653494 T1789 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 653497 T1789 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:51972
   [junit4]   2> 653498 T1789 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=51971, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/shard-1-001/cores, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/tempDir-001/jetty1, solrconfig=solrconfig.xml}
   [junit4]   2> 653498 T1789 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 653498 T1789 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/shard-1-001/'
   [junit4]   2> 653544 T1789 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/shard-1-001/solr.xml
   [junit4]   2> 653597 T1789 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/shard-1-001/cores
   [junit4]   2> 653597 T1789 oasc.CoreContainer.<init> New CoreContainer 1180552375
   [junit4]   2> 653597 T1789 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/shard-1-001/]
   [junit4]   2> 653598 T1789 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/shard-1-001/lib
   [junit4]   2> 653599 T1789 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/shard-1-001/lib).
   [junit4]   2> 653608 T1789 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 653608 T1789 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 653608 T1789 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 653609 T1789 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 653609 T1789 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 653609 T1789 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 653609 T1789 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 653610 T1789 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 653610 T1789 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 653610 T1789 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 653610 T1789 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 653611 T1789 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 653611 T1789 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 653611 T1789 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 653611 T1789 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 653612 T1789 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51949/solr
   [junit4]   2> 653612 T1789 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 653612 T1789 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 653613 T1789 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 653615 T1839 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4b2114be name:ZooKeeperConnection Watcher:127.0.0.1:51949 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 653615 T1789 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 653616 T1789 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 653618 T1789 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 653620 T1842 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@53594e77 name:ZooKeeperConnection Watcher:127.0.0.1:51949/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 653620 T1789 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 653624 T1789 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 654627 T1789 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51971_
   [junit4]   2> 654628 T1789 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:51971_
   [junit4]   2> 654631 T1789 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 654633 T1789 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/shard-1-001/cores
   [junit4]   2> 654636 T1789 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/shard-1-001/cores/collection1/, coreNodeName=, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/shard-1-001/cores/collection1, transient=false, loadOnStartup=true, config=solrconfig.xml, name=collection1, shard=, dataDir=data/}
   [junit4]   2> 654636 T1789 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/shard-1-001/cores/collection1/
   [junit4]   2> 654637 T1789 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 654638 T1843 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 654638 T1843 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 654638 T1843 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 654638 T1815 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 654639 T1816 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:51971_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51971",
   [junit4]   2> 	  "state":"down"} current state version: 4
   [junit4]   2> 654640 T1816 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:51971_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51971",
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 654640 T1816 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 654640 T1816 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 654641 T1815 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 654641 T1842 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 654641 T1827 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 655639 T1843 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 655640 T1843 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 655640 T1843 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 655640 T1843 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 655641 T1843 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 655641 T1843 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/shard-1-001/cores/collection1/'
   [junit4]   2> 655653 T1843 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 655655 T1843 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 655663 T1843 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 655681 T1843 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 655718 T1843 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 655819 T1843 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 655826 T1843 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 656086 T1843 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 656096 T1843 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 656098 T1843 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 656106 T1843 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 656109 T1843 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 656111 T1843 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 656112 T1843 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 656112 T1843 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 656112 T1843 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 656113 T1843 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 656113 T1843 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 656113 T1843 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 656113 T1843 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 656113 T1843 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 656114 T1843 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@347fd24a
   [junit4]   2> 656114 T1843 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/shard-1-001/cores/collection1/data
   [junit4]   2> 656115 T1843 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 656115 T1843 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 656115 T1843 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B62220C8632FEF-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 656115 T1843 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=49, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 656116 T1843 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2ba8fa9f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@62ddf31d),segFN=segments_1,generation=1}
   [junit4]   2> 656116 T1843 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 656121 T1843 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 656121 T1843 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 656121 T1843 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 656121 T1843 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 656122 T1843 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 656122 T1843 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 656122 T1843 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 656122 T1843 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 656123 T1843 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 656135 T1843 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 656136 T1843 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 656138 T1843 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 656139 T1843 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 656145 T1843 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /get,/admin/logging,/schema,/update/json/docs,/update/json,/admin/luke,/admin/file,/update,/admin/threads,/admin/segments,standard,/admin/plugins,/admin/mbeans,/update/csv,/config,/admin/system,/replication,/admin/ping,/admin/properties
   [junit4]   2> 656146 T1843 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 656147 T1843 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 656147 T1843 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 656148 T1843 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 656149 T1843 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 656150 T1843 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=11, maxMergeAtOnceExplicit=10, maxMergedSegmentMB=38.0224609375, floorSegmentMB=1.4755859375, forceMergeDeletesPctAllowed=27.85073561451421, segmentsPerTier=25.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7668826767996076
   [junit4]   2> 656150 T1843 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2ba8fa9f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@62ddf31d),segFN=segments_1,generation=1}
   [junit4]   2> 656150 T1843 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 656151 T1843 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@f6564de[collection1] main
   [junit4]   2> 656151 T1843 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 656151 T1843 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 656152 T1843 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 656152 T1843 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 656152 T1843 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 656152 T1843 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 656153 T1843 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 656153 T1843 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 656153 T1843 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 656153 T1843 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 656154 T1844 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@f6564de[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 656154 T1843 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 656155 T1847 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:51971 collection:collection1 shard:shard1
   [junit4]   2> 656155 T1789 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 656155 T1789 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 656156 T1847 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 656159 T1847 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 656160 T1815 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 656160 T1847 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 656160 T1847 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C252 name=collection1 org.apache.solr.core.SolrCore@8bae65b url=http://127.0.0.1:51971/collection1 node=127.0.0.1:51971_ C252_STATE=coll:collection1 core:collection1 props:{core=collection1, node_name=127.0.0.1:51971_, base_url=http://127.0.0.1:51971, state=down}
   [junit4]   2> 656160 T1847 C:collection1 S:shard1 C252 P51971 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:51971/collection1/
   [junit4]   2> 656160 T1816 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 656160 T1847 C:collection1 S:shard1 C252 P51971 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 656161 T1847 C:collection1 S:shard1 C252 P51971 oasc.SyncStrategy.syncToMe http://127.0.0.1:51971/collection1/ has no replicas
   [junit4]   2> 656161 T1847 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:51971/collection1/ shard1
   [junit4]   2> 656161 T1847 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 656161 T1815 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 656161 T1842 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 656161 T1827 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 656164 T1815 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 656164 T1816 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51971",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 656267 T1842 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 656267 T1827 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 656267 T1815 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 656316 T1847 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:51971/collection1/ and leader is http://127.0.0.1:51971/collection1/
   [junit4]   2> 656317 T1847 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:51971
   [junit4]   2> 656317 T1847 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 656317 T1847 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 656317 T1847 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 656318 T1815 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 656318 T1816 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:51971_",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51971",
   [junit4]   2> 	  "state":"active"} current state version: 7
   [junit4]   2> 656319 T1816 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:51971_",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51971",
   [junit4]   2> 	  "state":"active"}
   [junit4]   2> 656421 T1815 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 656421 T1827 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json

[...truncated too long message...]

]   2> 1462366 T8011 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:10256/io/q
   [junit4]   2> 1462366 T8011 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 9 connections to: http://127.0.0.1:10203/io/q
   [junit4]   2> 1462367 T8011 C:control_collection S:shard1 c:collection1 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 1462367 T8011 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:10242/io/q
   [junit4]   2> 1462367 T8011 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:10215/io/q
   [junit4]   2> 1462368 T8011 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:10256/io/q
   [junit4]   2> 1462368 T8011 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:10203/io/q
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=4B62220C8632FEF -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=nl_BE -Dtests.timezone=America/Maceio -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR    124s J0 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:10203/io/q/repfacttest_c8n_1x3_shard1_replica1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([4B62220C8632FEF:8CE21DFA669F4217]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:10203/io/q/repfacttest_c8n_1x3_shard1_replica1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 48 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 53 more
   [junit4]   2> 1462397 T8011 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4B62220C8632FEF-001
   [junit4]   2> 124345 T8010 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {intDefault=PostingsFormat(name=Memory doPackFST= false), id=FSTOrd50, multiDefault=PostingsFormat(name=LuceneFixedGap), _version_=PostingsFormat(name=Memory doPackFST= false), text=Lucene50(blocksize=128), a_t=PostingsFormat(name=Memory doPackFST= false), timestamp=PostingsFormat(name=Memory doPackFST= false), range_facet_l=PostingsFormat(name=Memory doPackFST= false)}, docValues:{timestamp=DocValuesFormat(name=Direct)}, sim=RandomSimilarityProvider(queryNorm=false,coord=no): {}, locale=nl_BE, timezone=America/Maceio
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=98202504,total=330825728
   [junit4]   2> NOTE: All tests run in this JVM: [ParsingFieldUpdateProcessorsTest, TestCoreDiscovery, TestMiniSolrCloudClusterSSL, SuggesterTest, PolyFieldTest, TestRawResponseWriter, TestClusterStateMutator, CacheHeaderTest, TestFoldingMultitermQuery, ExternalCollectionsTest, CursorPagingTest, TestLRUStatsCache, SliceStateTest, TestDynamicLoading, SpatialHeatmapFacetsTest, SolrPluginUtilsTest, ChangedSchemaMergeTest, TestFaceting, DistributedQueryElevationComponentTest, PrimUtilsTest, AddSchemaFieldsUpdateProcessorFactoryTest, TestRemoteStreaming, DateMathParserTest, TestSolrConfigHandlerCloud, TestCharFilters, TestRTGBase, TestLMJelinekMercerSimilarityFactory, TestCursorMarkWithoutUniqueKey, TestFieldCollectionResource, TestSchemaVersionResource, FieldMutatingUpdateProcessorTest, TestCollationField, TestSolr4Spatial2, CloudExitableDirectoryReaderTest, TestManagedResource, TestCSVResponseWriter, TestPHPSerializedResponseWriter, TestPseudoReturnFields, TestManagedSynonymFilterFactory, UUIDFieldTest, TestSolrQueryParserResource, HdfsDirectoryTest, SpatialFilterTest, TestCollapseQParserPlugin, TestConfigOverlay, TestArbitraryIndexDir, TestCryptoKeys, DistanceFunctionTest, TestSchemaManager, OverseerStatusTest, CurrencyFieldOpenExchangeTest, SearchHandlerTest, ZkSolrClientTest, TestFieldResource, HttpPartitionTest, CustomCollectionTest, ReplicationFactorTest]
   [junit4] Completed on J0 in 124.62s, 1 test, 1 error <<< FAILURES!

[...truncated 728 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 61 ignored (21 assumptions)

Total time: 53 minutes 25 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 208100971 bytes
Compression is 0.0%
Took 36 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2866 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2866/

4 tests failed.
REGRESSION:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
Didn't see all replicas for shard shard1 in c8n_1x2 come up within 30000 ms! ClusterState: {   "control_collection":{     "replicationFactor":"1",     "autoCreated":"true",     "shards":{"shard1":{         "range":"80000000-7fffffff",         "state":"active",         "replicas":{"core_node1":{             "node_name":"127.0.0.1:15547_",             "core":"collection1",             "state":"active",             "base_url":"http://127.0.0.1:15547",             "leader":"true"}}}},     "router":{"name":"compositeId"},     "maxShardsPerNode":"1",     "autoAddReplicas":"false"},   "collection1":{     "replicationFactor":"1",     "autoCreated":"true",     "shards":{       "shard1":{         "range":"80000000-ffffffff",         "state":"active",         "replicas":{"core_node2":{             "node_name":"127.0.0.1:56074_",             "core":"collection1",             "state":"active",             "base_url":"http://127.0.0.1:56074",             "leader":"true"}}},       "shard2":{         "range":"0-7fffffff",         "state":"active",         "replicas":{           "core_node1":{             "node_name":"127.0.0.1:30284_",             "core":"collection1",             "state":"active",             "base_url":"http://127.0.0.1:30284",             "leader":"true"},           "core_node3":{             "node_name":"127.0.0.1:56082_",             "core":"collection1",             "state":"active",             "base_url":"http://127.0.0.1:56082"}}}},     "router":{"name":"compositeId"},     "maxShardsPerNode":"1",     "autoAddReplicas":"false"},   "c8n_1x2":{     "replicationFactor":"2",     "shards":{"shard1":{         "range":"80000000-7fffffff",         "state":"active",         "replicas":{           "core_node1":{             "node_name":"127.0.0.1:30284_",             "core":"c8n_1x2_shard1_replica1",             "state":"recovering",             "base_url":"http://127.0.0.1:30284"},           "core_node2":{             "node_name":"127.0.0.1:56074_",             "core":"c8n_1x2_shard1_replica2",             "state":"active",             "base_url":"http://127.0.0.1:56074",             "leader":"true"}}}},     "router":{"name":"compositeId"},     "maxShardsPerNode":"1",     "autoAddReplicas":"false"}}

Stack Trace:
java.lang.AssertionError: Didn't see all replicas for shard shard1 in c8n_1x2 come up within 30000 ms! ClusterState: {
  "control_collection":{
    "replicationFactor":"1",
    "autoCreated":"true",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{"core_node1":{
            "node_name":"127.0.0.1:15547_",
            "core":"collection1",
            "state":"active",
            "base_url":"http://127.0.0.1:15547",
            "leader":"true"}}}},
    "router":{"name":"compositeId"},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false"},
  "collection1":{
    "replicationFactor":"1",
    "autoCreated":"true",
    "shards":{
      "shard1":{
        "range":"80000000-ffffffff",
        "state":"active",
        "replicas":{"core_node2":{
            "node_name":"127.0.0.1:56074_",
            "core":"collection1",
            "state":"active",
            "base_url":"http://127.0.0.1:56074",
            "leader":"true"}}},
      "shard2":{
        "range":"0-7fffffff",
        "state":"active",
        "replicas":{
          "core_node1":{
            "node_name":"127.0.0.1:30284_",
            "core":"collection1",
            "state":"active",
            "base_url":"http://127.0.0.1:30284",
            "leader":"true"},
          "core_node3":{
            "node_name":"127.0.0.1:56082_",
            "core":"collection1",
            "state":"active",
            "base_url":"http://127.0.0.1:56082"}}}},
    "router":{"name":"compositeId"},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false"},
  "c8n_1x2":{
    "replicationFactor":"2",
    "shards":{"shard1":{
        "range":"80000000-7fffffff",
        "state":"active",
        "replicas":{
          "core_node1":{
            "node_name":"127.0.0.1:30284_",
            "core":"c8n_1x2_shard1_replica1",
            "state":"recovering",
            "base_url":"http://127.0.0.1:30284"},
          "core_node2":{
            "node_name":"127.0.0.1:56074_",
            "core":"c8n_1x2_shard1_replica2",
            "state":"active",
            "base_url":"http://127.0.0.1:56074",
            "leader":"true"}}}},
    "router":{"name":"compositeId"},
    "maxShardsPerNode":"1",
    "autoAddReplicas":"false"}}
	at __randomizedtesting.SeedInfo.seed([4B98B9E495E556C7:C3CC863E3B193B3F]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.ensureAllReplicasAreActive(AbstractFullDistribZkTestBase.java:1925)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:238)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:105)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:43120/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:43120/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([4B98B9E495E556C7:C3CC863E3B193B3F]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:43501/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:43501/collection1
	at __randomizedtesting.SeedInfo.seed([4B98B9E495E556C7:C3CC863E3B193B3F]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:43501/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:43243/repfacttest_c8n_1x3_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:43243/repfacttest_c8n_1x3_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([4B98B9E495E556C7:C3CC863E3B193B3F]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:43243/repfacttest_c8n_1x3_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9871 lines...]
   [junit4] Suite: org.apache.solr.cloud.HttpPartitionTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/init-core-data-001
   [junit4]   2> 629736 T2198 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 629741 T2198 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 629741 T2199 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 629841 T2198 oasc.ZkTestServer.run start zk server on port:59767
   [junit4]   2> 629842 T2198 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 629842 T2198 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 629846 T2206 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@68f6cbbd name:ZooKeeperConnection Watcher:127.0.0.1:59767 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 629846 T2198 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 629846 T2198 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 629846 T2198 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 629848 T2198 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 629849 T2198 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 629850 T2209 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@12fa67aa name:ZooKeeperConnection Watcher:127.0.0.1:59767/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 629850 T2198 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 629850 T2198 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 629851 T2198 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 629852 T2198 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 629853 T2198 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 629854 T2198 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 629855 T2198 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 629856 T2198 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 629857 T2198 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 629858 T2198 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 629959 T2198 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 629960 T2198 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 629961 T2198 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 629962 T2198 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 629963 T2198 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 629963 T2198 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 629964 T2198 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 629965 T2198 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 629966 T2198 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 629967 T2198 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 629968 T2198 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 629968 T2198 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 629969 T2198 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 629970 T2198 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 629971 T2198 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 629971 T2198 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 629973 T2198 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 629973 T2198 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 631355 T2198 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/control-001/cores/collection1
   [junit4]   2> 631359 T2198 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 631373 T2198 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:33854
   [junit4]   2> 631373 T2198 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=15547, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/control-001/cores}
   [junit4]   2> 631374 T2198 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 631374 T2198 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/control-001/'
   [junit4]   2> 631403 T2198 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/control-001/solr.xml
   [junit4]   2> 631437 T2198 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/control-001/cores
   [junit4]   2> 631438 T2198 oasc.CoreContainer.<init> New CoreContainer 1529631768
   [junit4]   2> 631438 T2198 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/control-001/]
   [junit4]   2> 631438 T2198 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/control-001/lib
   [junit4]   2> 631439 T2198 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/control-001/lib).
   [junit4]   2> 631447 T2198 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 631447 T2198 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 631448 T2198 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 631448 T2198 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 631448 T2198 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 631448 T2198 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 631449 T2198 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 631449 T2198 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 631449 T2198 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 631449 T2198 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 631449 T2198 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 631450 T2198 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 631450 T2198 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 631450 T2198 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 631451 T2198 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 631451 T2198 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:59767/solr
   [junit4]   2> 631451 T2198 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 631451 T2198 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 631452 T2198 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 631454 T2221 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4a2c40f2 name:ZooKeeperConnection Watcher:127.0.0.1:59767 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 631455 T2198 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 631455 T2198 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 631457 T2198 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 631459 T2224 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@134e1605 name:ZooKeeperConnection Watcher:127.0.0.1:59767/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 631459 T2198 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 631460 T2198 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 631462 T2198 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 631464 T2198 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 631466 T2198 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 631468 T2198 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 631469 T2198 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 631471 T2198 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 631472 T2198 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 631473 T2198 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:15547_
   [junit4]   2> 631474 T2198 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:15547_
   [junit4]   2> 631475 T2198 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 631476 T2198 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 631478 T2198 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 631479 T2198 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:15547_
   [junit4]   2> 631479 T2198 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 631480 T2198 oasc.Overseer.start Overseer (id=93573816352702467-127.0.0.1:15547_-n_0000000000) starting
   [junit4]   2> 631482 T2198 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 631488 T2198 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 631488 T2226 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 631488 T2198 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 631488 T2225 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 631492 T2198 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/control-001/cores
   [junit4]   2> 631493 T2198 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/control-001/cores/collection1/, transient=false, name=collection1, collection=control_collection, loadOnStartup=true, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/control-001/cores/collection1, coreNodeName=, shard=, config=solrconfig.xml, dataDir=data/, schema=schema.xml}
   [junit4]   2> 631494 T2198 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/control-001/cores/collection1/
   [junit4]   2> 631494 T2198 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 631494 T2228 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 631495 T2228 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 631495 T2224 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 631495 T2228 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 631496 T2225 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:15547_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:15547",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state"} current state version: 0
   [junit4]   2> 631497 T2225 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:15547_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:15547",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 631497 T2225 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 631497 T2225 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 631498 T2224 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 632496 T2228 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 632497 T2228 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 632497 T2228 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 632498 T2228 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 632499 T2228 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 632499 T2228 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/control-001/cores/collection1/'
   [junit4]   2> 632516 T2228 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 632518 T2228 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 632529 T2228 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 632551 T2228 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 632598 T2228 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 632700 T2228 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 632707 T2228 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 632962 T2228 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 632973 T2228 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 632975 T2228 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 632983 T2228 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 632986 T2228 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 632988 T2228 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 632989 T2228 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 632989 T2228 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 632989 T2228 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 632990 T2228 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 632990 T2228 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 632990 T2228 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 632991 T2228 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 632991 T2228 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/control-001/cores/collection1/data/
   [junit4]   2> 632991 T2228 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4b8ca3ee
   [junit4]   2> 632992 T2228 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/control-001/cores/collection1/data
   [junit4]   2> 632992 T2228 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/control-001/cores/collection1/data/index/
   [junit4]   2> 632992 T2228 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 632992 T2228 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/control-001/cores/collection1/data/index
   [junit4]   2> 632993 T2228 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=47, maxMergeAtOnceExplicit=30, maxMergedSegmentMB=86.7578125, floorSegmentMB=0.20703125, forceMergeDeletesPctAllowed=21.07381196384413, segmentsPerTier=46.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8976432402508367
   [junit4]   2> 632993 T2228 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@848a197 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5868c4c0),segFN=segments_1,generation=1}
   [junit4]   2> 632993 T2228 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 632997 T2228 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 632997 T2228 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 632998 T2228 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 632998 T2228 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 632998 T2228 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 632998 T2228 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 632999 T2228 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 632999 T2228 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 632999 T2228 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 633011 T2228 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 633013 T2228 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 633014 T2228 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 633015 T2228 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 633019 T2228 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update/json,/admin/file,/admin/luke,/replication,/get,/update/json/docs,/config,/admin/properties,/admin/plugins,/admin/logging,/admin/ping,/update/csv,/admin/segments,/admin/mbeans,/admin/threads,/schema,standard,/admin/system,/update
   [junit4]   2> 633020 T2228 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 633021 T2228 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 633022 T2228 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 633023 T2228 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 633023 T2228 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 633023 T2228 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=13, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 633024 T2228 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@848a197 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5868c4c0),segFN=segments_1,generation=1}
   [junit4]   2> 633024 T2228 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 633024 T2228 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@4838d3f7[collection1] main
   [junit4]   2> 633024 T2228 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 633025 T2228 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 633025 T2228 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 633026 T2228 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 633026 T2228 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 633026 T2228 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 633026 T2228 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 633027 T2228 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 633027 T2228 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 633027 T2228 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 633028 T2229 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4838d3f7[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 633028 T2228 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 633029 T2232 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:15547 collection:control_collection shard:shard1
   [junit4]   2> 633029 T2198 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 633030 T2198 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 633030 T2232 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 633033 T2198 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 633034 T2198 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 633034 T2232 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 633036 T2236 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@29d4ee86 name:ZooKeeperConnection Watcher:127.0.0.1:59767/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 633036 T2198 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 633036 T2224 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 633036 T2198 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 633037 T2232 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 633037 T2198 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 633037 T2225 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 633037 T2232 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C281 name=collection1 org.apache.solr.core.SolrCore@586922d2 url=http://127.0.0.1:15547/collection1 node=127.0.0.1:15547_ C281_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:15547_, core=collection1, state=down, base_url=http://127.0.0.1:15547}
   [junit4]   2> 633038 T2232 C:control_collection S:shard1 C281 P15547 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:15547/collection1/
   [junit4]   2> 633039 T2232 C:control_collection S:shard1 C281 P15547 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 633039 T2232 C:control_collection S:shard1 C281 P15547 oasc.SyncStrategy.syncToMe http://127.0.0.1:15547/collection1/ has no replicas
   [junit4]   2> 633039 T2236 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 633039 T2232 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:15547/collection1/ shard1
   [junit4]   2> 633039 T2224 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 633040 T2232 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 633040 T2198 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 633043 T2224 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 633044 T2225 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:15547",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 633147 T2224 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 633147 T2236 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 633196 T2232 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:15547/collection1/ and leader is http://127.0.0.1:15547/collection1/
   [junit4]   2> 633197 T2232 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:15547
   [junit4]   2> 633197 T2232 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 633197 T2232 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 633197 T2232 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 633198 T2224 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 633199 T2225 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:15547_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:15547",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state"} current state version: 3
   [junit4]   2> 633200 T2225 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:15547_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:15547",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 633302 T2236 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 633302 T2224 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 634281 T2198 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/shard-1-001/cores/collection1
   [junit4]   2> 634283 T2198 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/shard-1-001
   [junit4]   2> 634283 T2198 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 634286 T2198 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:60073
   [junit4]   2> 634286 T2198 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solrconfig=solrconfig.xml, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/tempDir-001/jetty1, hostPort=30284, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/shard-1-001/cores, hostContext=/}
   [junit4]   2> 634286 T2198 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 634287 T2198 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/shard-1-001/'
   [junit4]   2> 634316 T2198 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/shard-1-001/solr.xml
   [junit4]   2> 634358 T2198 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/shard-1-001/cores
   [junit4]   2> 634358 T2198 oasc.CoreContainer.<init> New CoreContainer 1169737037
   [junit4]   2> 634358 T2198 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/shard-1-001/]
   [junit4]   2> 634359 T2198 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/shard-1-001/lib
   [junit4]   2> 634359 T2198 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/shard-1-001/lib).
   [junit4]   2> 634368 T2198 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 634368 T2198 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 634368 T2198 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 634368 T2198 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 634369 T2198 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 634369 T2198 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 634369 T2198 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 634369 T2198 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 634370 T2198 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 634370 T2198 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 634370 T2198 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 634370 T2198 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 634371 T2198 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 634371 T2198 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 634371 T2198 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 634371 T2198 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:59767/solr
   [junit4]   2> 634372 T2198 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 634372 T2198 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 634373 T2198 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 634374 T2248 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@18a60dd6 name:ZooKeeperConnection Watcher:127.0.0.1:59767 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 634374 T2198 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 634375 T2198 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 634377 T2198 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 634378 T2251 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6b62d4a8 name:ZooKeeperConnection Watcher:127.0.0.1:59767/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 634378 T2198 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 634381 T2198 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 635385 T2198 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:30284_
   [junit4]   2> 635386 T2198 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:30284_
   [junit4]   2> 635389 T2198 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 635391 T2198 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/shard-1-001/cores
   [junit4]   2> 635393 T2198 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, dataDir=data/, transient=false, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/shard-1-001/cores/collection1, collection=collection1, name=collection1, shard=, loadOnStartup=true, coreNodeName=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/shard-1-001/cores/collection1/}
   [junit4]   2> 635393 T2198 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/shard-1-001/cores/collection1/
   [junit4]   2> 635393 T2198 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 635394 T2252 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 635394 T2252 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 635395 T2224 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 635395 T2252 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 635396 T2225 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:30284_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:30284",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "operation":"state"} current state version: 4
   [junit4]   2> 635396 T2225 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:30284_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:30284",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 635397 T2225 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 635397 T2225 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 635398 T2251 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 635398 T2236 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 635398 T2224 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 636395 T2252 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 636396 T2252 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 636397 T2252 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 636397 T2252 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 636398 T2252 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 636398 T2252 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/shard-1-001/cores/collection1/'
   [junit4]   2> 636415 T2252 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 636419 T2252 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 636432 T2252 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 636465 T2252 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 636540 T2252 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 636641 T2252 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 636661 T2252 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 637140 T2252 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 637157 T2252 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 637160 T2252 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 637171 T2252 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 637175 T2252 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 637179 T2252 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 637180 T2252 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 637180 T2252 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 637181 T2252 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 637181 T2252 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 637182 T2252 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 637182 T2252 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 637182 T2252 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 637182 T2252 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 637183 T2252 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4b8ca3ee
   [junit4]   2> 637184 T2252 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/shard-1-001/cores/collection1/data
   [junit4]   2> 637184 T2252 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 637184 T2252 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 637185 T2252 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.HttpPartitionTest 4B98B9E495E556C7-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 637185 T2252 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=47, maxMergeAtOnceExplicit=30, maxMergedSegmentMB=86.7578125, floorSegmentMB=0.20703125, forceMergeDeletesPctAllowed=21.07381196384413, segmentsPerTier=46.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8976432402508367
   [junit4]   2> 637186 T2252 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5d784686 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@35e375b),segFN=segments_1,generation=1}
   [junit4]   2> 637186 T2252 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 637193 T2252 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 637193 T2252 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 637193 T2252 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 637194 T2252 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 637194 T2252 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 637194 T2252 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 637195 T2252 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 637195 T2252 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 637196 T2252 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 637215 T2252 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 637218 T2252 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 637220 T2252 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 637222 T2252 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 637231 T2252 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update/json,/admin/file,/admin/luke,/replication,/get,/update/json/docs,/config,/admin/properties,/admin/plugins,/admin/logging,/admin/ping,/update/csv,/admin/segments,/admin/mbeans,/admin/threads,/schema,standard,/admin/system,/update
   [junit4]   2> 637233 T2252 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 637235 T2252 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 637235 T2252 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 637237 T2252 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 637237 T2252 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 637238 T2252 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=13, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 637239 T2252 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5d784686 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@35e375b),segFN=segments_1,generation=1}
   [junit4]   2> 637239 T2252 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 637239 T2252 C:collection1 c:collection1

[...truncated too long message...]

on1 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> 1520542 T3657 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:43487 43487
   [junit4]   2> 1520644 T3658 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:43487 43487
   [junit4]   2> 1520645 T3658 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1520645 T3657 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 6 connections to: http://127.0.0.1:43501/
   [junit4]   2> 1520645 T3657 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:43515/
   [junit4]   2> 1520645 T3657 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:43493/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=4B98B9E495E556C7 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=hi_IN -Dtests.timezone=Asia/Tbilisi -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   70.3s J1 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:43501/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([4B98B9E495E556C7:C3CC863E3B193B3F]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:43501/collection1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 49 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 54 more
   [junit4]   2> 1520658 T3657 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 4B98B9E495E556C7-001
   [junit4]   2> 70339 T3656 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Lucene50, sim=DefaultSimilarity, locale=hi_IN, timezone=Asia/Tbilisi
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=54634576,total=290979840
   [junit4]   2> NOTE: All tests run in this JVM: [EchoParamsTest, StatelessScriptUpdateProcessorFactoryTest, PreAnalyzedUpdateProcessorTest, DocumentAnalysisRequestHandlerTest, BasicDistributedZkTest, LeaderFailoverAfterPartitionTest, BufferStoreTest, HdfsSyncSliceTest, TestOrdValues, BJQParserTest, CacheHeaderTest, DateRangeFieldTest, NumericFieldsTest, TestSimpleQParserPlugin, TestBlendedInfixSuggestions, LeaderElectionTest, LukeRequestHandlerTest, TestValueSourceCache, PeerSyncTest, SolrTestCaseJ4Test, SolrIndexSplitterTest, TestRangeQuery, ExternalCollectionsTest, TestRemoteStreaming, TestManagedSynonymFilterFactory, TestJettySolrRunner, RemoteQueryErrorTest, DefaultValueUpdateProcessorTest, ResponseLogComponentTest, TestRecoveryHdfs, SharedFSAutoReplicaFailoverUtilsTest, TestRequestStatusCollectionAPI, DirectUpdateHandlerTest, ShowFileRequestHandlerTest, TestImplicitCoreProperties, ZkStateWriterTest, RegexBoostProcessorTest, TestStressReorder, TestDocSet, CSVRequestHandlerTest, TestSolrJ, TestLeaderElectionZkExpiry, TestZkChroot, FieldAnalysisRequestHandlerTest, TestCollectionAPI, TestFieldTypeCollectionResource, TestSolrDeletionPolicy2, BadComponentTest, IndexSchemaTest, TestCollationField, TestFastOutputStream, SignatureUpdateProcessorFactoryTest, DistributedQueryElevationComponentTest, DistanceUnitsTest, TestIntervalFaceting, FastVectorHighlighterTest, HdfsDirectoryTest, TestHighFrequencyDictionaryFactory, TestShortCircuitedRequests, CollectionsAPIDistributedZkTest, MultiThreadedOCPTest, TestBlobHandler, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J1 in 71.14s, 1 test, 1 error <<< FAILURES!

[...truncated 701 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 1 failure, 61 ignored (21 assumptions)

Total time: 52 minutes 44 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 229460883 bytes
Compression is 0.0%
Took 41 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2865 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2865/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:19336/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19336/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([E208874E82B190B3:6A5CB8942C4DFD4B]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([E208874E82B190B3:6A5CB8942C4DFD4B]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([E208874E82B190B3:6A5CB8942C4DFD4B]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9691 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/init-core-data-001
   [junit4]   2> 588837 T2627 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 588842 T2627 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 588843 T2628 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 588943 T2627 oasc.ZkTestServer.run start zk server on port:29308
   [junit4]   2> 588944 T2627 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 588945 T2627 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 588948 T2635 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1d1cb489 name:ZooKeeperConnection Watcher:127.0.0.1:29308 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 588949 T2627 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 588949 T2627 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 588949 T2627 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 588952 T2627 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 588953 T2627 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 588954 T2638 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@25fee9d9 name:ZooKeeperConnection Watcher:127.0.0.1:29308/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 588954 T2627 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 588954 T2627 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 588954 T2627 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 588956 T2627 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 588957 T2627 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 588958 T2627 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 588959 T2627 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 588959 T2627 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 588961 T2627 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 588962 T2627 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 589063 T2627 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 589064 T2627 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 589066 T2627 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 589066 T2627 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 589068 T2627 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 589069 T2627 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 589070 T2627 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 589071 T2627 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 589073 T2627 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 589073 T2627 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 589075 T2627 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 589076 T2627 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 589077 T2627 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 589078 T2627 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 589080 T2627 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 589080 T2627 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 589082 T2627 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 589083 T2627 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 590319 T2627 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/control-001/cores/collection1
   [junit4]   2> 590322 T2627 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 590328 T2627 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:29312
   [junit4]   2> 590328 T2627 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/tempDir-001/control/data, hostPort=29311, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/control-001/cores}
   [junit4]   2> 590328 T2627 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 590329 T2627 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/control-001/'
   [junit4]   2> 590358 T2627 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/control-001/solr.xml
   [junit4]   2> 590393 T2627 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/control-001/cores
   [junit4]   2> 590393 T2627 oasc.CoreContainer.<init> New CoreContainer 261011770
   [junit4]   2> 590393 T2627 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/control-001/]
   [junit4]   2> 590393 T2627 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/control-001/lib
   [junit4]   2> 590394 T2627 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/control-001/lib).
   [junit4]   2> 590402 T2627 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 590403 T2627 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 590403 T2627 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 590403 T2627 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 590403 T2627 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 590403 T2627 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 590404 T2627 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 590404 T2627 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 590404 T2627 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 590404 T2627 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 590405 T2627 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 590405 T2627 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 590405 T2627 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 590406 T2627 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 590406 T2627 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 590406 T2627 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:29308/solr
   [junit4]   2> 590406 T2627 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 590406 T2627 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 590407 T2627 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 590409 T2650 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@61eddf17 name:ZooKeeperConnection Watcher:127.0.0.1:29308 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 590410 T2627 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 590410 T2627 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 590412 T2627 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 590413 T2653 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@73f87c7a name:ZooKeeperConnection Watcher:127.0.0.1:29308/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 590413 T2627 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 590414 T2627 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 590416 T2627 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 590417 T2627 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 590418 T2627 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 590420 T2627 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 590421 T2627 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 590422 T2627 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 590423 T2627 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 590424 T2627 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:29311_
   [junit4]   2> 590425 T2627 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:29311_
   [junit4]   2> 590426 T2627 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 590427 T2627 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 590427 T2627 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 590429 T2627 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:29311_
   [junit4]   2> 590429 T2627 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 590430 T2627 oasc.Overseer.start Overseer (id=93572682300063747-127.0.0.1:29311_-n_0000000000) starting
   [junit4]   2> 590431 T2627 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 590435 T2627 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 590436 T2655 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 590436 T2627 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 590436 T2654 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 590439 T2627 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/control-001/cores
   [junit4]   2> 590440 T2627 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=control_collection, loadOnStartup=true, coreNodeName=, shard=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/control-001/cores/collection1/, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/control-001/cores/collection1, dataDir=data/, name=collection1, transient=false, config=solrconfig.xml}
   [junit4]   2> 590441 T2627 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/control-001/cores/collection1/
   [junit4]   2> 590441 T2627 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 590441 T2657 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 590442 T2657 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 590442 T2657 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 590442 T2653 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 590443 T2654 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:29311",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:29311_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"} current state version: 0
   [junit4]   2> 590443 T2654 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:29311",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:29311_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 590444 T2654 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 590444 T2654 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 590445 T2653 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 591442 T2657 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 591443 T2657 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 591444 T2657 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 591444 T2657 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 591444 T2657 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 591445 T2657 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/control-001/cores/collection1/'
   [junit4]   2> 591463 T2657 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 591465 T2657 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 591476 T2657 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 591498 T2657 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 591551 T2657 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 591652 T2657 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 591660 T2657 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 591939 T2657 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 591950 T2657 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 591952 T2657 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 591960 T2657 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 591963 T2657 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 591966 T2657 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 591967 T2657 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 591967 T2657 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 591967 T2657 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 591968 T2657 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 591968 T2657 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 591968 T2657 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 591968 T2657 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 591969 T2657 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/control-001/cores/collection1/data/
   [junit4]   2> 591969 T2657 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6df68f29
   [junit4]   2> 591969 T2657 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/control-001/cores/collection1/data
   [junit4]   2> 591970 T2657 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/control-001/cores/collection1/data/index/
   [junit4]   2> 591970 T2657 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 591970 T2657 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/control-001/cores/collection1/data/index
   [junit4]   2> 591970 T2657 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=34, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 591971 T2657 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@288be08c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@77431c1f),segFN=segments_1,generation=1}
   [junit4]   2> 591971 T2657 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 591975 T2657 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 591975 T2657 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 591976 T2657 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 591976 T2657 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 591976 T2657 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 591976 T2657 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 591977 T2657 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 591977 T2657 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 591977 T2657 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 591990 T2657 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 591991 T2657 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 591993 T2657 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 591994 T2657 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 591998 T2657 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/plugins,/get,/replication,/update,/update/json/docs,/admin/threads,/admin/properties,standard,/admin/ping,/admin/file,/admin/logging,/admin/luke,/admin/segments,/config,/admin/system,/update/csv,/update/json,/schema,/admin/mbeans
   [junit4]   2> 592000 T2657 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 592001 T2657 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 592001 T2657 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 592002 T2657 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 592002 T2657 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 592003 T2657 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=44, maxMergeAtOnceExplicit=32, maxMergedSegmentMB=20.4443359375, floorSegmentMB=1.7490234375, forceMergeDeletesPctAllowed=23.407085415817107, segmentsPerTier=33.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 592003 T2657 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@288be08c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@77431c1f),segFN=segments_1,generation=1}
   [junit4]   2> 592003 T2657 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 592004 T2657 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@214b6a60[collection1] main
   [junit4]   2> 592004 T2657 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 592005 T2657 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 592005 T2657 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 592005 T2657 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 592006 T2657 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 592006 T2657 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 592006 T2657 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 592006 T2657 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 592006 T2657 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 592007 T2657 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 592007 T2657 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 592008 T2658 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@214b6a60[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 592008 T2661 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:29311 collection:control_collection shard:shard1
   [junit4]   2> 592008 T2627 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 592009 T2627 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 592009 T2661 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 592012 T2627 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 592013 T2661 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 592013 T2627 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 592015 T2665 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@68a5528e name:ZooKeeperConnection Watcher:127.0.0.1:29308/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 592015 T2653 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 592015 T2661 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 592015 T2661 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 592015 T2627 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 592016 T2654 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 592016 T2627 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> ASYNC  NEW_CORE C746 name=collection1 org.apache.solr.core.SolrCore@6621bf6b url=http://127.0.0.1:29311/collection1 node=127.0.0.1:29311_ C746_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:29311, core=collection1, node_name=127.0.0.1:29311_, state=down}
   [junit4]   2> 592015 T2661 C:control_collection S:shard1 C746 P29311 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:29311/collection1/
   [junit4]   2> 592016 T2627 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 592017 T2661 C:control_collection S:shard1 C746 P29311 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 592018 T2653 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 592018 T2661 C:control_collection S:shard1 C746 P29311 oasc.SyncStrategy.syncToMe http://127.0.0.1:29311/collection1/ has no replicas
   [junit4]   2> 592018 T2661 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:29311/collection1/ shard1
   [junit4]   2> 592019 T2661 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 592020 T2627 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 592021 T2627 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 592021 T2627 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 592022 T2627 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 592023 T2668 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7fafffa name:ZooKeeperConnection Watcher:127.0.0.1:29308/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 592023 T2653 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 592023 T2627 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 592024 T2654 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:29311",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 592024 T2627 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 592027 T2654 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 592027 T2654 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 592028 T2653 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 592028 T2665 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 592031 T2653 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 592032 T2654 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 592033 T2665 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 592033 T2653 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 592074 T2661 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:29311/collection1/ and leader is http://127.0.0.1:29311/collection1/
   [junit4]   2> 592075 T2661 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:29311
   [junit4]   2> 592075 T2661 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 592075 T2661 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 592075 T2661 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 592077 T2653 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 592077 T2654 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:29311",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:29311_",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"} current state version: 4
   [junit4]   2> 592078 T2654 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:29311",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:29311_",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 592181 T2665 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 592181 T2653 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 593300 T2627 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/shard-1-001/cores/collection1
   [junit4]   2> 593302 T2627 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/shard-1-001
   [junit4]   2> 593303 T2627 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 593306 T2627 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:29318
   [junit4]   2> 593306 T2627 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=29317, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/shard-1-001/cores, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/tempDir-001/jetty1, solrconfig=solrconfig.xml}
   [junit4]   2> 593307 T2627 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 593307 T2627 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/shard-1-001/'
   [junit4]   2> 593338 T2627 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/shard-1-001/solr.xml
   [junit4]   2> 593374 T2627 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/shard-1-001/cores
   [junit4]   2> 593375 T2627 oasc.CoreContainer.<init> New CoreContainer 2016949994
   [junit4]   2> 593375 T2627 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/shard-1-001/]
   [junit4]   2> 593375 T2627 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/shard-1-001/lib
   [junit4]   2> 593376 T2627 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/shard-1-001/lib).
   [junit4]   2> 593386 T2627 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 593386 T2627 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 593387 T2627 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 593387 T2627 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 593387 T2627 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 593387 T2627 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 593387 T2627 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 593388 T2627 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 593388 T2627 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 593388 T2627 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 593388 T2627 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 593389 T2627 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 593389 T2627 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 593389 T2627 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 593390 T2627 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 593390 T2627 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:29308/solr
   [junit4]   2> 593390 T2627 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 593390 T2627 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 593391 T2627 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 593392 T2680 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@56013843 name:ZooKeeperConnection Watcher:127.0.0.1:29308 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 593393 T2627 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 593394 T2627 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 593396 T2627 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 593396 T2683 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1f872fec name:ZooKeeperConnection Watcher:127.0.0.1:29308/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 593397 T2627 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 593400 T2627 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 594403 T2627 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:29317_
   [junit4]   2> 594405 T2627 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:29317_
   [junit4]   2> 594408 T2627 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 594410 T2627 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/shard-1-001/cores
   [junit4]   2> 594412 T2627 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {transient=false, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/shard-1-001/cores/collection1/, dataDir=data/, loadOnStartup=true, collection=collection1, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/shard-1-001/cores/collection1, schema=schema.xml, name=collection1, coreNodeName=, shard=}
   [junit4]   2> 594413 T2627 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/shard-1-001/cores/collection1/
   [junit4]   2> 594413 T2627 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 594415 T2684 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 594415 T2684 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 594416 T2653 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 594416 T2684 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 594416 T2684 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 594417 T2654 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:29317",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:29317_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"} current state version: 5
   [junit4]   2> 594417 T2684 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 594417 T2654 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:29317",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:29317_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 594417 T2654 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 594417 T2654 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 594418 T2684 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 594418 T2684 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 594418 T2654 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 594419 T2683 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 594419 T2683 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 595418 T2684 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 595419 T2684 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 595419 T2684 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 595420 T2684 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 595420 T2684 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 595421 T2684 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/shard-1-001/cores/collection1/'
   [junit4]   2> 595439 T2684 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 595441 T2684 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 595452 T2684 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 595473 T2684 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 595523 T2684 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 595624 T2684 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 595636 T2684 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 596043 T2684 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 596058 T2684 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 596062 T2684 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 596071 T2684 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 596075 T2684 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 596079 T2684 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 596079 T2684 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 596080 T2684 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 596080 T2684 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 596080 T2684 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 596081 T2684 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 596081 T2684 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 596081 T2684 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 596082 T2684 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 596082 T2684 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6df68f29
   [junit4]   2> 596083 T2684 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/shard-1-001/cores/collection1/data
   [junit4]   2> 596083 T2684 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 596083 T2684 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 596084 T2684 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E208874E82B190B3-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 596084 T2684 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=34, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 596085 T2684 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b27d066 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@62e2de42),segFN=segments_1,generation=1}
   [junit4]   2> 596085 T2684 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 596092 T2684 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 596092 T2684 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 596093 T2684 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 596093 T2684 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 596093 T2684 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 596093 T2684 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 596094 T2684 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 596094 T2684 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 596094 T2684 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 596114 T2684 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 596119 T2684 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 596122 T2684 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 596126 T2684 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 596136 T2684 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/plugins,/get,/replication,/update,/update/json/docs,/admin/threads,/admin/properties,standard,/admin/ping,/admin/file,/admin/logging,/admin/luke,/admin/segments,/config,/admin/system,/update/csv,/update/json,/schema,/admin/mbeans
   [junit4]   2> 596139 T2684 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 596140 T2684 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 596141 T2684 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 596142 T2684 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 596143 T2684 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 596143 T2684 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=44, maxMergeAtOnceExplicit=32, maxMergedSegmentMB=20.4443359375, floorSegmentMB=1.7490234375, forceMergeDeletesPctAllowed=23.407085415817107, segmentsPerTier=33.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 596144 T2684 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b27d066 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@62e2de42),segFN=segments_1,generation=1}
   [junit4]   2> 596144 T2684 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 596145 T2684 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@4cd05b0d[collection1] main
   [junit4]   2> 596145 T2684 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 596146 T2684 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 596146 T2684 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 596146 T2684 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 596147 T2684 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 596147 T2684 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 596147 T2684 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 596148 T2684 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 596148 T2684 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 596148 T2684 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 596149 T2685 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4cd05b0d[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 596150 T2684 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 596151 T2688 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:29317 collection:collection1 shard:shard1
   [junit4]   2> 596151 T2627 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 596151 T2627 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 596151 T2688 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 596156 T2688 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 596157 T2653 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 596157 T2688 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 596158 T2688 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C747 name=collection1 org.apache.solr.core.SolrCore@7ed485eb url=http://127.0.0.1:29317/collection1 node=127.0.0.1:29317_ C747_STATE=coll:collection1 core:collection1 props:{base_url=http://127.0.0.1:29317, core=collection1, node_name=127.0.0.1:29317_, state=down}
   [junit4]   2> 596158 T2688 C:collection1 S:shard1 C747 P29317 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:29317/collection1/
   [junit4]   2> 596158 T2654 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 596158 T2688 C:collection1 S:shard1 C747 P29317 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 596159 T2688 C:collection1 S:shard1 C747 P29317 oasc.SyncStrategy.syncToMe http://127.0.0.1:29317/collection1/ has no replicas
   [junit4]   2> 596159 T2688 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:29317/collection1/ shard1
   [junit4]   2> 596159 T2688 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 596159 T2654 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 596160 T2683 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 596161 T2683 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 596163 T2653 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 596164 T2654 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:29317",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 596165 T2654 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 596165 T2683 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 596166 T2683 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 596214 T2688 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:29317/collection1/ and leader is http://127.0.0.1:29317/collection1/
   [junit4]   2> 596215 T2688 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:29317
   [junit4]   2> 596215 T2688 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 596215 T2688 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 596215 T2688 C:collection1 S:shard1 c:co

[...truncated too long message...]

N Closing 2 connections to: http://127.0.0.1:44948/f/b
   [junit4]   2> 1548672 T4551 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:44933/f/b
   [junit4]   2> 1548672 T4551 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:44911/f/b
   [junit4]   2> 1548673 T4551 C:control_collection S:shard1 c:collection1 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 1548673 T4551 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:44902/f/b
   [junit4]   2> 1548673 T4551 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:44948/f/b
   [junit4]   2> 1548674 T4551 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:44933/f/b
   [junit4]   2> 1548674 T4551 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:44911/f/b
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=E208874E82B190B3 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=nl_BE -Dtests.timezone=Etc/GMT-14 -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   77.7s J2 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([E208874E82B190B3:6A5CB8942C4DFD4B]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 1548699 T4551 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E208874E82B190B3-001
   [junit4]   2> 77696 T4550 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {_version_=BlockTreeOrds(blocksize=128), a_t=PostingsFormat(name=LuceneFixedGap), timestamp=PostingsFormat(name=LuceneFixedGap), text=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), multiDefault=PostingsFormat(name=LuceneFixedGap), id=FSTOrd50, intDefault=BlockTreeOrds(blocksize=128), range_facet_l=PostingsFormat(name=LuceneFixedGap)}, docValues:{timestamp=DocValuesFormat(name=Lucene50)}, sim=RandomSimilarityProvider(queryNorm=false,coord=no): {}, locale=nl_BE, timezone=Etc/GMT-14
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=108721312,total=294649856
   [junit4]   2> NOTE: All tests run in this JVM: [XmlUpdateRequestHandlerTest, TestLuceneMatchVersion, LeaderElectionTest, TestReplicationHandler, TestMacros, HdfsBasicDistributedZkTest, TestHighFrequencyDictionaryFactory, TestDistributedMissingSort, TestPerFieldSimilarity, IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestReversedWildcardFilterFactory, HdfsBasicDistributedZk2Test, DocExpirationUpdateProcessorFactoryTest, BinaryUpdateRequestHandlerTest, SearchHandlerTest, TestSolrConfigHandlerConcurrent, TestSearcherReuse, TestAddFieldRealTimeGet, TestExactStatsCache, HdfsCollectionsAPIDistributedZkTest, CoreMergeIndexesAdminHandlerTest, TestIndexSearcher, StressHdfsTest, SpatialFilterTest, TestFastLRUCache, TestDFRSimilarityFactory, TestUtils, AliasIntegrationTest, ParsingFieldUpdateProcessorsTest, TestComponentsName, RAMDirectoryFactoryTest, TestManagedStopFilterFactory, ShardRoutingTest, TestTrackingShardHandlerFactory, TestFieldTypeResource, TestRawResponseWriter, DirectSolrSpellCheckerTest, CircularListTest, TestSortingResponseWriter, TestManagedSchema, QueryResultKeyTest, ResourceLoaderTest, SyncSliceTest, TestNoOpRegenerator, TestBinaryField, TestLazyCores, SuggestComponentTest, DistributedDebugComponentTest, RequiredFieldsTest, TestClusterStateMutator, PolyFieldTest, TestComplexPhraseQParserPlugin, TestNamedUpdateProcessors, OverseerRolesTest, HdfsLockFactoryTest, SuggesterTSTTest, BadComponentTest, OverseerTest, TestOmitPositions, ChaosMonkeyNothingIsSafeTest, CollectionsAPIDistributedZkTest, UnloadDistributedZkTest, ReplicationFactorTest]
   [junit4] Completed on J2 in 78.75s, 1 test, 1 error <<< FAILURES!

[...truncated 711 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 62 ignored (22 assumptions)

Total time: 53 minutes 28 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 208390757 bytes
Compression is 0.0%
Took 37 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2864 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2864/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:64623/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:64623/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([9F43DFEF91BABBC7:1717E0353F46D63F]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:14387/yeu/o/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:14387/yeu/o/collection1
	at __randomizedtesting.SeedInfo.seed([9F43DFEF91BABBC7:1717E0353F46D63F]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:14387/yeu/o/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:21753/_n/v/repfacttest_c8n_1x3_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:21753/_n/v/repfacttest_c8n_1x3_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([9F43DFEF91BABBC7:1717E0353F46D63F]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:21753/_n/v/repfacttest_c8n_1x3_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9452 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/init-core-data-001
   [junit4]   2> 328603 T1654 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /yeu/o
   [junit4]   2> 328607 T1654 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 328608 T1655 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 328708 T1654 oasc.ZkTestServer.run start zk server on port:14378
   [junit4]   2> 328709 T1654 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 328709 T1654 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 328712 T1662 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@764cf769 name:ZooKeeperConnection Watcher:127.0.0.1:14378 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 328712 T1654 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 328713 T1654 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 328713 T1654 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 328715 T1654 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 328715 T1654 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 328717 T1665 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@313f20fb name:ZooKeeperConnection Watcher:127.0.0.1:14378/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 328717 T1654 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 328717 T1654 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 328717 T1654 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 328719 T1654 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 328721 T1654 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 328722 T1654 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 328723 T1654 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 328724 T1654 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 328726 T1654 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 328726 T1654 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 328728 T1654 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 328729 T1654 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 328730 T1654 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 328731 T1654 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 328732 T1654 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 328733 T1654 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 328734 T1654 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 328734 T1654 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 328736 T1654 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 328736 T1654 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 328738 T1654 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 328738 T1654 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 328740 T1654 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 328740 T1654 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 328742 T1654 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 328742 T1654 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 328744 T1654 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 328744 T1654 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 329684 T1654 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/control-001/cores/collection1
   [junit4]   2> 329687 T1654 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 329691 T1654 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:14382
   [junit4]   2> 329692 T1654 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/control-001/cores, hostPort=14381, hostContext=/yeu/o}
   [junit4]   2> 329692 T1654 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 329693 T1654 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/control-001/'
   [junit4]   2> 329722 T1654 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/control-001/solr.xml
   [junit4]   2> 329757 T1654 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/control-001/cores
   [junit4]   2> 329757 T1654 oasc.CoreContainer.<init> New CoreContainer 1244522727
   [junit4]   2> 329758 T1654 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/control-001/]
   [junit4]   2> 329758 T1654 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/control-001/lib
   [junit4]   2> 329759 T1654 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/control-001/lib).
   [junit4]   2> 329767 T1654 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 329767 T1654 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 329768 T1654 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 329768 T1654 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 329768 T1654 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 329768 T1654 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 329769 T1654 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 329769 T1654 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 329769 T1654 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 329769 T1654 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 329769 T1654 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 329770 T1654 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 329770 T1654 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 329770 T1654 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 329771 T1654 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 329771 T1654 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:14378/solr
   [junit4]   2> 329771 T1654 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 329771 T1654 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 329772 T1654 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 329774 T1677 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2020b0dd name:ZooKeeperConnection Watcher:127.0.0.1:14378 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 329774 T1654 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 329774 T1654 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 329776 T1654 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 329777 T1680 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@66818a66 name:ZooKeeperConnection Watcher:127.0.0.1:14378/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 329777 T1654 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 329778 T1654 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 329780 T1654 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 329782 T1654 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 329784 T1654 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 329786 T1654 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 329788 T1654 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 329789 T1654 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 329791 T1654 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 329792 T1654 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:14381_yeu%2Fo
   [junit4]   2> 329792 T1654 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:14381_yeu%2Fo
   [junit4]   2> 329794 T1654 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 329795 T1654 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 329797 T1654 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 329798 T1654 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:14381_yeu%2Fo
   [junit4]   2> 329798 T1654 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 329800 T1654 oasc.Overseer.start Overseer (id=93571648746815491-127.0.0.1:14381_yeu%2Fo-n_0000000000) starting
   [junit4]   2> 329801 T1654 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 329807 T1654 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 329808 T1682 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 329808 T1654 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 329808 T1681 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 329811 T1654 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/control-001/cores
   [junit4]   2> 329813 T1654 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {dataDir=data/, loadOnStartup=true, collection=control_collection, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/control-001/cores/collection1, shard=, coreNodeName=, transient=false, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/control-001/cores/collection1/, name=collection1, schema=schema.xml}
   [junit4]   2> 329813 T1654 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/control-001/cores/collection1/
   [junit4]   2> 329813 T1654 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 329814 T1684 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 329814 T1684 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 329815 T1680 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 329815 T1684 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 329815 T1681 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:14381_yeu%2Fo",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14381/yeu/o"} current state version: 0
   [junit4]   2> 329816 T1681 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:14381_yeu%2Fo",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14381/yeu/o"}
   [junit4]   2> 329816 T1681 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 329816 T1681 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 329817 T1680 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 330815 T1684 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 330816 T1684 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 330816 T1684 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 330817 T1684 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 330817 T1684 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 330817 T1684 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/control-001/cores/collection1/'
   [junit4]   2> 330829 T1684 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 330831 T1684 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 330839 T1684 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 330854 T1684 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 330887 T1684 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 330888 T1684 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 330895 T1684 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 331166 T1684 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 331176 T1684 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 331178 T1684 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 331188 T1684 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 331192 T1684 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 331194 T1684 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 331195 T1684 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 331195 T1684 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 331196 T1684 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 331196 T1684 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 331197 T1684 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 331197 T1684 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 331197 T1684 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 331197 T1684 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/control-001/cores/collection1/data/
   [junit4]   2> 331197 T1684 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1e0a90
   [junit4]   2> 331198 T1684 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/control-001/cores/collection1/data
   [junit4]   2> 331198 T1684 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/control-001/cores/collection1/data/index/
   [junit4]   2> 331198 T1684 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 331199 T1684 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/control-001/cores/collection1/data/index
   [junit4]   2> 331199 T1684 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=47, maxMergedSegmentMB=82.7080078125, floorSegmentMB=2.0263671875, forceMergeDeletesPctAllowed=29.945163504008303, segmentsPerTier=22.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 331200 T1684 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@210b0272 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@59cd9b50),segFN=segments_1,generation=1}
   [junit4]   2> 331200 T1684 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 331204 T1684 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 331204 T1684 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 331204 T1684 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 331205 T1684 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 331205 T1684 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 331205 T1684 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 331205 T1684 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 331206 T1684 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 331206 T1684 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 331218 T1684 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 331219 T1684 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 331221 T1684 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 331222 T1684 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 331227 T1684 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/ping,/admin/file,/update,/admin/segments,/get,/admin/system,/update/json/docs,standard,/admin/luke,/update/csv,/replication,/admin/plugins,/config,/admin/logging,/admin/threads,/admin/mbeans,/schema,/admin/properties,/update/json
   [junit4]   2> 331228 T1684 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 331230 T1684 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 331230 T1684 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 331231 T1684 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 331231 T1684 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 331231 T1684 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=39, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 331232 T1684 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@210b0272 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@59cd9b50),segFN=segments_1,generation=1}
   [junit4]   2> 331232 T1684 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 331232 T1684 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@68c0bab3[collection1] main
   [junit4]   2> 331233 T1684 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 331234 T1684 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 331234 T1684 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 331234 T1684 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 331234 T1684 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 331234 T1684 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 331235 T1684 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 331235 T1684 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 331235 T1684 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 331236 T1684 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 331236 T1684 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 331236 T1685 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@68c0bab3[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 331237 T1688 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:14381/yeu/o collection:control_collection shard:shard1
   [junit4]   2> 331237 T1654 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 331237 T1654 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 331238 T1688 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 331241 T1654 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 331241 T1654 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 331242 T1688 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 331243 T1692 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1a788550 name:ZooKeeperConnection Watcher:127.0.0.1:14378/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 331243 T1654 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 331243 T1680 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 331243 T1654 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 331244 T1688 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 331244 T1688 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 331244 T1654 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> ASYNC  NEW_CORE C229 name=collection1 org.apache.solr.core.SolrCore@43a615ea url=http://127.0.0.1:14381/yeu/o/collection1 node=127.0.0.1:14381_yeu%2Fo C229_STATE=coll:control_collection core:collection1 props:{core=collection1, node_name=127.0.0.1:14381_yeu%2Fo, state=down, base_url=http://127.0.0.1:14381/yeu/o}
   [junit4]   2> 331244 T1688 C:control_collection S:shard1 C229 P14381 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:14381/yeu/o/collection1/
   [junit4]   2> 331244 T1681 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 331245 T1688 C:control_collection S:shard1 C229 P14381 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 331245 T1688 C:control_collection S:shard1 C229 P14381 oasc.SyncStrategy.syncToMe http://127.0.0.1:14381/yeu/o/collection1/ has no replicas
   [junit4]   2> 331246 T1688 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:14381/yeu/o/collection1/ shard1
   [junit4]   2> 331246 T1688 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 331247 T1654 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 331247 T1654 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 331248 T1654 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 331248 T1654 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 331250 T1695 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@28c6d8f7 name:ZooKeeperConnection Watcher:127.0.0.1:14378/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 331250 T1680 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 331250 T1654 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 331251 T1654 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 331251 T1681 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14381/yeu/o",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 1
   [junit4]   2> 331253 T1681 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 1
   [junit4]   2> 331254 T1681 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 331254 T1680 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 331254 T1692 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 331258 T1680 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 331259 T1681 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 331260 T1680 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 331260 T1692 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 331300 T1688 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:14381/yeu/o/collection1/ and leader is http://127.0.0.1:14381/yeu/o/collection1/
   [junit4]   2> 331301 T1688 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:14381/yeu/o
   [junit4]   2> 331301 T1688 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 331301 T1688 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 331301 T1688 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 331302 T1680 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 331303 T1681 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:14381_yeu%2Fo",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14381/yeu/o",
   [junit4]   2> 	  "core_node_name":"core_node1"} current state version: 3
   [junit4]   2> 331304 T1681 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:14381_yeu%2Fo",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14381/yeu/o",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 331407 T1680 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 331407 T1692 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 332582 T1654 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/shard-1-001/cores/collection1
   [junit4]   2> 332585 T1654 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/shard-1-001
   [junit4]   2> 332585 T1654 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 332588 T1654 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:14388
   [junit4]   2> 332588 T1654 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/tempDir-001/jetty1, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/shard-1-001/cores, hostContext=/yeu/o, solrconfig=solrconfig.xml, hostPort=14387}
   [junit4]   2> 332589 T1654 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 332589 T1654 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/shard-1-001/'
   [junit4]   2> 332624 T1654 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/shard-1-001/solr.xml
   [junit4]   2> 332666 T1654 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/shard-1-001/cores
   [junit4]   2> 332667 T1654 oasc.CoreContainer.<init> New CoreContainer 1558603718
   [junit4]   2> 332667 T1654 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/shard-1-001/]
   [junit4]   2> 332667 T1654 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/shard-1-001/lib
   [junit4]   2> 332668 T1654 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/shard-1-001/lib).
   [junit4]   2> 332677 T1654 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 332677 T1654 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 332678 T1654 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 332678 T1654 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 332678 T1654 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 332678 T1654 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 332679 T1654 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 332679 T1654 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 332679 T1654 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 332679 T1654 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 332679 T1654 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 332680 T1654 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 332680 T1654 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 332680 T1654 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 332681 T1654 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 332681 T1654 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:14378/solr
   [junit4]   2> 332681 T1654 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 332681 T1654 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 332682 T1654 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 332684 T1707 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@78281bad name:ZooKeeperConnection Watcher:127.0.0.1:14378 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 332684 T1654 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 332684 T1654 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 332686 T1654 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 332687 T1710 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4838ded7 name:ZooKeeperConnection Watcher:127.0.0.1:14378/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 332687 T1654 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 332691 T1654 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 333696 T1654 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:14387_yeu%2Fo
   [junit4]   2> 333696 T1654 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:14387_yeu%2Fo
   [junit4]   2> 333699 T1654 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 333701 T1654 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/shard-1-001/cores
   [junit4]   2> 333704 T1654 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {transient=false, name=collection1, collection=collection1, loadOnStartup=true, schema=schema.xml, dataDir=data/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/shard-1-001/cores/collection1, config=solrconfig.xml, shard=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/shard-1-001/cores/collection1/, coreNodeName=}
   [junit4]   2> 333704 T1654 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/shard-1-001/cores/collection1/
   [junit4]   2> 333704 T1654 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 333706 T1711 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 333706 T1711 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 333707 T1680 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 333708 T1711 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 333708 T1711 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 333708 T1681 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:14387_yeu%2Fo",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14387/yeu/o"} current state version: 4
   [junit4]   2> 333709 T1711 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 333709 T1681 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:14387_yeu%2Fo",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14387/yeu/o"}
   [junit4]   2> 333709 T1681 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 333709 T1681 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 333709 T1711 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 333710 T1711 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 333710 T1681 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 333711 T1710 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 333711 T1710 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 334710 T1711 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 334711 T1711 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 334711 T1711 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 334712 T1711 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 334712 T1711 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 334712 T1711 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/shard-1-001/cores/collection1/'
   [junit4]   2> 334725 T1711 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 334728 T1711 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 334737 T1711 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 334755 T1711 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 334808 T1711 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 334809 T1711 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 334817 T1711 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 335140 T1711 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 335155 T1711 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 335157 T1711 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 335165 T1711 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 335169 T1711 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 335172 T1711 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 335173 T1711 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 335173 T1711 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 335174 T1711 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 335174 T1711 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 335174 T1711 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 335175 T1711 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 335175 T1711 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 335175 T1711 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 335175 T1711 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1e0a90
   [junit4]   2> 335176 T1711 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/shard-1-001/cores/collection1/data
   [junit4]   2> 335176 T1711 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 335176 T1711 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 335177 T1711 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 9F43DFEF91BABBC7-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 335177 T1711 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=47, maxMergedSegmentMB=82.7080078125, floorSegmentMB=2.0263671875, forceMergeDeletesPctAllowed=29.945163504008303, segmentsPerTier=22.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 335178 T1711 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@14a57204 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@79e06f32),segFN=segments_1,generation=1}
   [junit4]   2> 335178 T1711 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 335183 T1711 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 335184 T1711 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 335184 T1711 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 335184 T1711 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 335184 T1711 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 335184 T1711 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 335185 T1711 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 335185 T1711 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 335185 T1711 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 335210 T1711 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 335212 T1711 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 335214 T1711 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 335215 T1711 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 335222 T1711 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/ping,/admin/file,/update,/admin/segments,/get,/admin/system,/update/json/docs,standard,/admin/luke,/update/csv,/replication,/admin/plugins,/config,/admin/logging,/admin/threads,/admin/mbeans,/schema,/admin/properties,/update/json
   [junit4]   2> 335224 T1711 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 335225 T1711 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 335226 T1711 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 335227 T1711 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 335227 T1711 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 335228 T1711 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=39, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 335228 T1711 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@14a57204 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@79e06f32),segFN=segments_1,generation=1}
   [junit4]   2> 335229 T1711 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 335229 T1711 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@66f12493[collection1] main
   [junit4]   2> 335229 T1711 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 335230 T1711 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 335230 T1711 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 335231 T1711 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 335231 T1711 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 335231 T1711 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 335231 T1711 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 335232 T1711 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 335232 T1711 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 335232 T1711 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 335233 T1712 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@66f12493[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 335233 T1711 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 335234 T1715 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:14387/yeu/o collection:collection1 shard:shard1
   [junit4]   2> 335234 T1654 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 335234 T1654 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 335234 T1715 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 335238 T1715 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 335239 T1680 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 335239 T1715 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 335240 T1715 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C230 name=collection1 org.apache.solr.core.SolrCore@2a09ce06 url=http://127.0.0.1:14387/yeu/o/collection1 node=127.0.0.1:14387_yeu%2Fo C230_STATE=coll:collection1 core:collection1 props:{core=collection1, node_name=127.0.0.1:14387_yeu%2Fo, state=down, base_url=http://127.0.0.1:14387/yeu/o}
   [junit4]   2> 335240 T1715 C:collection1 S:shard1 C230 P14387 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:14387/yeu/o/collection1/
   [junit4]   2> 335240 T1715 C:collection1 S:shard1 C230 P14387 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 335240 T1681 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 4
   [junit4]   2> 335240 T1715 C:collection1 S:shard1 C230 P14387 oasc.SyncStrategy.syncToMe http://127.0.0.1:14387/yeu/o/collection1/ has no replicas
   [junit4]   2> 335240 T1715 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:14387/yeu/o/collection1/ shard1
   [junit4]   2> 335241 T1715 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 335241 T1681 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 335242 T1710 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 335243 T1710 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 335244 T1681 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14387/yeu/o",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 4
   [junit4]   2> 335245 T1681 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 335246 T1710 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 335246 T1710 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 335248 T1680 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 335295 T1715 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:14387/yeu/o/collection1/ and leader is http://127.0.0.1:14387/yeu/o/collection1/
   [junit4]   2> 335296 T1715 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:14387/yeu/o
   [junit4]   2> 335296 T1715 C:colle

[...truncated too long message...]

pache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:486)
   [junit4]   2> 		at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:455)
   [junit4]   2> 		at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:442)
   [junit4]   2> 		at org.apache.solr.cloud.ShardLeaderElectionContextBase$1.execute(ElectionContext.java:146)
   [junit4]   2> 		at org.apache.solr.common.util.RetryUtil.retryOnThrowable(RetryUtil.java:31)
   [junit4]   2> 		at org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:142)
   [junit4]   2> 		... 11 more
   [junit4]   2> 	
   [junit4]   2> 1122025 T1419 C:control_collection S:shard1 c:collection1 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> 1122057 T1419 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:64604 64604
   [junit4]   2> 1122266 T1420 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:64604 64604
   [junit4]   2> 1122268 T1420 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/aliases.json
   [junit4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 		4	/solr/collections/collection1/state.json
   [junit4]   2> 		3	/solr/collections/c8n_1x3_commits/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 		4	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1122268 T1419 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:64623/
   [junit4]   2> 1122268 T1419 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:64607/
   [junit4]   2> 1122269 T1419 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:64617/
   [junit4]   2> 1122269 T1419 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:64613/
   [junit4]   2> 1122270 T1419 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:64629/
   [junit4]   2> 1122270 T1419 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:64623/
   [junit4]   2> 1122271 T1419 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:64607/
   [junit4]   2> 1122271 T1419 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:64617/
   [junit4]   2> 1122271 T1419 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:64613/
   [junit4]   2> 1122272 T1419 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:64629/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderInitiatedRecoveryOnCommitTest -Dtests.method=test -Dtests.seed=9F43DFEF91BABBC7 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=th_TH_TH_#u-nu-thai -Dtests.timezone=America/Indiana/Marengo -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   75.8s J0 | LeaderInitiatedRecoveryOnCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:64623/c8n_1x3_commits_shard1_replica3
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([9F43DFEF91BABBC7:1717E0353F46D63F]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 48 more
   [junit4]   2> 1122346 T1419 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 9F43DFEF91BABBC7-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=DefaultSimilarity, locale=th_TH_TH_#u-nu-thai, timezone=America/Indiana/Marengo
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=107844592,total=266338304
   [junit4]   2> NOTE: All tests run in this JVM: [AlternateDirectoryTest, TestFastWriter, TestDistributedMissingSort, CurrencyFieldXmlFileTest, BadCopyFieldTest, DeleteShardTest, UpdateParamsTest, TestWordDelimiterFilterFactory, TestValueSourceCache, TestDistribDocBasedVersion, TestLazyCores, TestHighlightDedupGrouping, TestPivotHelperCode, TestBinaryResponseWriter, DistributedFacetPivotWhiteBoxTest, HttpPartitionTest, TestDefaultSearchFieldResource, DeleteLastCustomShardedReplicaTest, TestExactStatsCache, ClusterStateTest, DistributedQueueTest, TestReversedWildcardFilterFactory, LeaderInitiatedRecoveryOnCommitTest]
   [junit4] Completed on J0 in 75.87s, 1 test, 1 error <<< FAILURES!

[...truncated 898 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 62 ignored (22 assumptions)

Total time: 54 minutes 26 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 216298799 bytes
Compression is 0.0%
Took 45 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2863 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2863/

5 tests failed.
REGRESSION:  org.apache.solr.core.TestDynamicLoading.testDynamicLoading

Error Message:
Could not get expected value  '2' for path 'overlay/runtimeLib/colltest/version' full output: {   "responseHeader":{     "status":0,     "QTime":1},   "overlay":{     "znodeVersion":2,     "runtimeLib":{"colltest":{         "name":"colltest",         "version":1}},     "requestHandler":{       "/test1":{         "name":"/test1",         "class":"org.apache.solr.core.BlobStoreTestRequestHandler",         "runtimeLib":true},       "/runtime":{         "name":"/runtime",         "class":"org.apache.solr.core.RuntimeLibReqHandler",         "runtimeLib":true}},     "searchComponent":{"get":{         "name":"get",         "class":"org.apache.solr.core.RuntimeLibSearchComponent",         "runtimeLib":true}},     "queryResponseWriter":{"json1":{         "name":"json1",         "class":"org.apache.solr.core.RuntimeLibResponseWriter",         "runtimeLib":true}}}}

Stack Trace:
java.lang.AssertionError: Could not get expected value  '2' for path 'overlay/runtimeLib/colltest/version' full output: {
  "responseHeader":{
    "status":0,
    "QTime":1},
  "overlay":{
    "znodeVersion":2,
    "runtimeLib":{"colltest":{
        "name":"colltest",
        "version":1}},
    "requestHandler":{
      "/test1":{
        "name":"/test1",
        "class":"org.apache.solr.core.BlobStoreTestRequestHandler",
        "runtimeLib":true},
      "/runtime":{
        "name":"/runtime",
        "class":"org.apache.solr.core.RuntimeLibReqHandler",
        "runtimeLib":true}},
    "searchComponent":{"get":{
        "name":"get",
        "class":"org.apache.solr.core.RuntimeLibSearchComponent",
        "runtimeLib":true}},
    "queryResponseWriter":{"json1":{
        "name":"json1",
        "class":"org.apache.solr.core.RuntimeLibResponseWriter",
        "runtimeLib":true}}}}
	at __randomizedtesting.SeedInfo.seed([124C9216793140FD:CA01BF418EECE55D]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.core.TestSolrConfigHandler.testForResponseElement(TestSolrConfigHandler.java:406)
	at org.apache.solr.core.TestDynamicLoading.testDynamicLoading(TestDynamicLoading.java:212)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.solr.schema.TestCloudManagedSchemaConcurrent.test

Error Message:
QUERY FAILED: xpath=/response/lst[@name='responseHeader']/int[@name='status'][.='0']  request=/schema/copyfields/?wt=xml&updateTimeoutSecs=15  response=<?xml version="1.0" encoding="UTF-8"?> <response>  <lst name="responseHeader">   <int name="status">500</int>   <int name="QTime">28</int> </lst> <lst name="error">   <str name="msg">copyField source :'copySource0' is not a glob and doesn't match any explicit field or dynamicField.</str>   <str name="trace">org.apache.solr.common.SolrException: copyField source :'copySource0' is not a glob and doesn't match any explicit field or dynamicField.  at org.apache.solr.schema.IndexSchema.registerCopyField(IndexSchema.java:899)  at org.apache.solr.schema.IndexSchema.registerCopyField(IndexSchema.java:821)  at org.apache.solr.schema.ManagedIndexSchema.addCopyFields(ManagedIndexSchema.java:773)  at org.apache.solr.rest.schema.CopyFieldCollectionResource.post(CopyFieldCollectionResource.java:176)  at org.restlet.resource.ServerResource.doHandle(ServerResource.java:437)  at org.restlet.resource.ServerResource.doConditionalHandle(ServerResource.java:350)  at org.restlet.resource.ServerResource.handle(ServerResource.java:952)  at org.restlet.resource.Finder.handle(Finder.java:246)  at org.restlet.routing.Filter.doHandle(Filter.java:159)  at org.restlet.routing.Filter.handle(Filter.java:206)  at org.restlet.routing.Router.doHandle(Router.java:431)  at org.restlet.routing.Router.handle(Router.java:648)  at org.restlet.routing.Filter.doHandle(Filter.java:159)  at org.restlet.routing.Filter.handle(Filter.java:206)  at org.restlet.routing.Filter.doHandle(Filter.java:159)  at org.restlet.routing.Filter.handle(Filter.java:206)  at org.restlet.routing.Filter.doHandle(Filter.java:159)  at org.restlet.engine.application.StatusFilter.doHandle(StatusFilter.java:155)  at org.restlet.routing.Filter.handle(Filter.java:206)  at org.restlet.routing.Filter.doHandle(Filter.java:159)  at org.restlet.routing.Filter.handle(Filter.java:206)  at org.restlet.engine.CompositeHelper.handle(CompositeHelper.java:211)  at org.restlet.engine.application.ApplicationHelper.handle(ApplicationHelper.java:84)  at org.restlet.Application.handle(Application.java:381)  at org.restlet.routing.Filter.doHandle(Filter.java:159)  at org.restlet.routing.Filter.handle(Filter.java:206)  at org.restlet.routing.Router.doHandle(Router.java:431)  at org.restlet.routing.Router.handle(Router.java:648)  at org.restlet.routing.Filter.doHandle(Filter.java:159)  at org.restlet.routing.Filter.handle(Filter.java:206)  at org.restlet.routing.Router.doHandle(Router.java:431)  at org.restlet.routing.Router.handle(Router.java:648)  at org.restlet.routing.Filter.doHandle(Filter.java:159)  at org.restlet.routing.Filter.handle(Filter.java:206)  at org.restlet.engine.CompositeHelper.handle(CompositeHelper.java:211)  at org.restlet.Component.handle(Component.java:392)  at org.restlet.Server.handle(Server.java:516)  at org.restlet.engine.ServerHelper.handle(ServerHelper.java:72)  at org.restlet.engine.adapter.HttpServerHelper.handle(HttpServerHelper.java:152)  at org.restlet.ext.servlet.ServerServlet.service(ServerServlet.java:1089)  at javax.servlet.http.HttpServlet.service(HttpServlet.java:848)  at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:669)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:457)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)  at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)  at org.eclipse.jetty.server.Dispatcher.forward(Dispatcher.java:276)  at org.eclipse.jetty.server.Dispatcher.forward(Dispatcher.java:103)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:102)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)  at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)  at org.eclipse.jetty.server.Server.handle(Server.java:368)  at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)  at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)  at org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:953)  at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1014)  at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:861)  at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240)  at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)  at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)  at java.lang.Thread.run(Thread.java:745) </str>   <int name="code">500</int> </lst> </response> 

Stack Trace:
java.lang.AssertionError: QUERY FAILED: xpath=/response/lst[@name='responseHeader']/int[@name='status'][.='0']  request=/schema/copyfields/?wt=xml&updateTimeoutSecs=15  response=<?xml version="1.0" encoding="UTF-8"?>
<response>

<lst name="responseHeader">
  <int name="status">500</int>
  <int name="QTime">28</int>
</lst>
<lst name="error">
  <str name="msg">copyField source :'copySource0' is not a glob and doesn't match any explicit field or dynamicField.</str>
  <str name="trace">org.apache.solr.common.SolrException: copyField source :'copySource0' is not a glob and doesn't match any explicit field or dynamicField.
	at org.apache.solr.schema.IndexSchema.registerCopyField(IndexSchema.java:899)
	at org.apache.solr.schema.IndexSchema.registerCopyField(IndexSchema.java:821)
	at org.apache.solr.schema.ManagedIndexSchema.addCopyFields(ManagedIndexSchema.java:773)
	at org.apache.solr.rest.schema.CopyFieldCollectionResource.post(CopyFieldCollectionResource.java:176)
	at org.restlet.resource.ServerResource.doHandle(ServerResource.java:437)
	at org.restlet.resource.ServerResource.doConditionalHandle(ServerResource.java:350)
	at org.restlet.resource.ServerResource.handle(ServerResource.java:952)
	at org.restlet.resource.Finder.handle(Finder.java:246)
	at org.restlet.routing.Filter.doHandle(Filter.java:159)
	at org.restlet.routing.Filter.handle(Filter.java:206)
	at org.restlet.routing.Router.doHandle(Router.java:431)
	at org.restlet.routing.Router.handle(Router.java:648)
	at org.restlet.routing.Filter.doHandle(Filter.java:159)
	at org.restlet.routing.Filter.handle(Filter.java:206)
	at org.restlet.routing.Filter.doHandle(Filter.java:159)
	at org.restlet.routing.Filter.handle(Filter.java:206)
	at org.restlet.routing.Filter.doHandle(Filter.java:159)
	at org.restlet.engine.application.StatusFilter.doHandle(StatusFilter.java:155)
	at org.restlet.routing.Filter.handle(Filter.java:206)
	at org.restlet.routing.Filter.doHandle(Filter.java:159)
	at org.restlet.routing.Filter.handle(Filter.java:206)
	at org.restlet.engine.CompositeHelper.handle(CompositeHelper.java:211)
	at org.restlet.engine.application.ApplicationHelper.handle(ApplicationHelper.java:84)
	at org.restlet.Application.handle(Application.java:381)
	at org.restlet.routing.Filter.doHandle(Filter.java:159)
	at org.restlet.routing.Filter.handle(Filter.java:206)
	at org.restlet.routing.Router.doHandle(Router.java:431)
	at org.restlet.routing.Router.handle(Router.java:648)
	at org.restlet.routing.Filter.doHandle(Filter.java:159)
	at org.restlet.routing.Filter.handle(Filter.java:206)
	at org.restlet.routing.Router.doHandle(Router.java:431)
	at org.restlet.routing.Router.handle(Router.java:648)
	at org.restlet.routing.Filter.doHandle(Filter.java:159)
	at org.restlet.routing.Filter.handle(Filter.java:206)
	at org.restlet.engine.CompositeHelper.handle(CompositeHelper.java:211)
	at org.restlet.Component.handle(Component.java:392)
	at org.restlet.Server.handle(Server.java:516)
	at org.restlet.engine.ServerHelper.handle(ServerHelper.java:72)
	at org.restlet.engine.adapter.HttpServerHelper.handle(HttpServerHelper.java:152)
	at org.restlet.ext.servlet.ServerServlet.service(ServerServlet.java:1089)
	at javax.servlet.http.HttpServlet.service(HttpServlet.java:848)
	at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:669)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:457)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
	at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
	at org.eclipse.jetty.server.Dispatcher.forward(Dispatcher.java:276)
	at org.eclipse.jetty.server.Dispatcher.forward(Dispatcher.java:103)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:395)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:220)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:102)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
	at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
	at org.eclipse.jetty.server.Server.handle(Server.java:368)
	at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
	at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
	at org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:953)
	at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1014)
	at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:861)
	at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240)
	at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
	at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
	at java.lang.Thread.run(Thread.java:745)
</str>
  <int name="code">500</int>
</lst>
</response>

	at __randomizedtesting.SeedInfo.seed([124C9216793140FD:9A18ADCCD7CD2D05]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.schema.TestCloudManagedSchemaConcurrent.verifySuccess(TestCloudManagedSchemaConcurrent.java:107)
	at org.apache.solr.schema.TestCloudManagedSchemaConcurrent.copyField(TestCloudManagedSchemaConcurrent.java:153)
	at org.apache.solr.schema.TestCloudManagedSchemaConcurrent.access$500(TestCloudManagedSchemaConcurrent.java:45)
	at org.apache.solr.schema.TestCloudManagedSchemaConcurrent$Operation$5.execute(TestCloudManagedSchemaConcurrent.java:320)
	at org.apache.solr.schema.TestCloudManagedSchemaConcurrent.concurrentOperationsTest(TestCloudManagedSchemaConcurrent.java:450)
	at org.apache.solr.schema.TestCloudManagedSchemaConcurrent.test(TestCloudManagedSchemaConcurrent.java:252)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:16194/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:16194/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([124C9216793140FD:9A18ADCCD7CD2D05]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([124C9216793140FD:9A18ADCCD7CD2D05]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([124C9216793140FD:9A18ADCCD7CD2D05]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9331 lines...]
   [junit4] Suite: org.apache.solr.schema.TestCloudManagedSchemaConcurrent
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.schema.TestCloudManagedSchemaConcurrent 124C9216793140FD-001/init-core-data-001
   [junit4]   2> 184718 T310 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 184718 T310 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 184724 T310 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 184725 T311 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 184825 T310 oasc.ZkTestServer.run start zk server on port:33182
   [junit4]   2> 184826 T310 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 184827 T310 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 184830 T318 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@34a9e6b3 name:ZooKeeperConnection Watcher:127.0.0.1:33182 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 184830 T310 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 184831 T310 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 184831 T310 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 184834 T310 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 184835 T310 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 184836 T321 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4e5cc007 name:ZooKeeperConnection Watcher:127.0.0.1:33182/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 184837 T310 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 184837 T310 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 184837 T310 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 184839 T310 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 184841 T310 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 184842 T310 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 184843 T310 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-managed-schema.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 184844 T310 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 184846 T310 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 184847 T310 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 184849 T310 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 184849 T310 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 184850 T310 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 184851 T310 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 184852 T310 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 184853 T310 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 184854 T310 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 184855 T310 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 184856 T310 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 184857 T310 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 184858 T310 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 184859 T310 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 184861 T310 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 184861 T310 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 184862 T310 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 184863 T310 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 184864 T310 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 184865 T310 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 186287 T310 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.schema.TestCloudManagedSchemaConcurrent 124C9216793140FD-001/control-001/cores/collection1
   [junit4]   2> 186290 T310 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 186294 T310 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:33185
   [junit4]   2> 186294 T310 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.schema.TestCloudManagedSchemaConcurrent 124C9216793140FD-001/control-001/cores, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.schema.TestCloudManagedSchemaConcurrent 124C9216793140FD-001/tempDir-001/control/data, hostPort=33185}
   [junit4]   2> 186295 T310 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 186295 T310 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.schema.TestCloudManagedSchemaConcurrent 124C9216793140FD-001/control-001/'
   [junit4]   2> 186326 T310 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.schema.TestCloudManagedSchemaConcurrent 124C9216793140FD-001/control-001/solr.xml
   [junit4]   2> 186387 T310 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.schema.TestCloudManagedSchemaConcurrent 124C9216793140FD-001/control-001/cores
   [junit4]   2> 186388 T310 oasc.CoreContainer.<init> New CoreContainer 2110834955
   [junit4]   2> 186388 T310 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.schema.TestCloudManagedSchemaConcurrent 124C9216793140FD-001/control-001/]
   [junit4]   2> 186389 T310 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.schema.TestCloudManagedSchemaConcurrent 124C9216793140FD-001/control-001/lib
   [junit4]   2> 186390 T310 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.schema.TestCloudManagedSchemaConcurrent 124C9216793140FD-001/control-001/lib).
   [junit4]   2> 186403 T310 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 186404 T310 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 186404 T310 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 186405 T310 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 186405 T310 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 186405 T310 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 186406 T310 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 186406 T310 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 186406 T310 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 186407 T310 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 186407 T310 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 186408 T310 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 186408 T310 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 186409 T310 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 186409 T310 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 186409 T310 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:33182/solr
   [junit4]   2> 186410 T310 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 186410 T310 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 186411 T310 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 186413 T333 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@61411465 name:ZooKeeperConnection Watcher:127.0.0.1:33182 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 186414 T310 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 186415 T310 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 186417 T310 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 186419 T336 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7511f43f name:ZooKeeperConnection Watcher:127.0.0.1:33182/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 186419 T310 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 186421 T310 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 186423 T310 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 186425 T310 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 186428 T310 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 186430 T310 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 186433 T310 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 186434 T310 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 186436 T310 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 186438 T310 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33185_
   [junit4]   2> 186439 T310 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:33185_
   [junit4]   2> 186440 T310 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 186442 T310 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 186443 T310 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 186445 T310 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:33185_
   [junit4]   2> 186445 T310 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 186447 T310 oasc.Overseer.start Overseer (id=93570696784510979-127.0.0.1:33185_-n_0000000000) starting
   [junit4]   2> 186449 T310 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 186455 T310 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 186455 T338 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 186455 T310 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 186456 T337 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 186460 T310 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.schema.TestCloudManagedSchemaConcurrent 124C9216793140FD-001/control-001/cores
   [junit4]   2> 186461 T310 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {loadOnStartup=true, name=collection1, config=solrconfig.xml, transient=false, shard=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.schema.TestCloudManagedSchemaConcurrent 124C9216793140FD-001/control-001/cores/collection1/, collection=control_collection, coreNodeName=, dataDir=data/, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.schema.TestCloudManagedSchemaConcurrent 124C9216793140FD-001/control-001/cores/collection1}
   [junit4]   2> 186462 T310 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.schema.TestCloudManagedSchemaConcurrent 124C9216793140FD-001/control-001/cores/collection1/
   [junit4]   2> 186462 T310 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 186463 T340 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 186463 T340 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 186464 T340 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 186464 T336 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 186465 T337 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:33185_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33185",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null} current state version: 0
   [junit4]   2> 186466 T337 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:33185_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33185",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 186466 T337 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 186467 T337 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 186468 T336 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 187464 T340 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 187465 T340 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 187466 T340 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 187466 T340 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 187467 T340 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 187467 T340 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.schema.TestCloudManagedSchemaConcurrent 124C9216793140FD-001/control-001/cores/collection1/'
   [junit4]   2> 187485 T340 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 187487 T340 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 187499 T340 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 187519 T340 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 187561 T340 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 187562 T340 C:control_collection c:collection1 oass.ManagedIndexSchemaFactory.create The schema is configured as managed, but managed schema resource managed-schema not found - loading non-managed schema schema.xml instead
   [junit4]   2> 187563 T340 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 187578 T340 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 187963 T340 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 187978 T340 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 187981 T340 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 188007 T340 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 188012 T340 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 188016 T340 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 188017 T340 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 188018 T340 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 188018 T340 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 188019 T340 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 188019 T340 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 188028 T340 C:control_collection c:collection1 oass.ManagedIndexSchema.persistManagedSchemaToZooKeeper Created and persisted managed schema znode at /configs/conf1/managed-schema
   [junit4]   2> 188029 T340 C:control_collection c:collection1 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml.bak
   [junit4]   2> 188032 T340 C:control_collection c:collection1 oass.ManagedIndexSchemaFactory.zkUgradeToManagedSchema After upgrading to managed schema in ZooKeeper, renamed the non-managed schema /configs/conf1/schema.xml to /configs/conf1/schema.xml.bak
   [junit4]   2> 188032 T340 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 188033 T340 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory solr.NRTCachingDirectoryFactory
   [junit4]   2> 188034 T340 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.schema.TestCloudManagedSchemaConcurrent 124C9216793140FD-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.schema.TestCloudManagedSchemaConcurrent 124C9216793140FD-001/control-001/cores/collection1/data/
   [junit4]   2> 188034 T340 C:control_collection c:collection1 oasc.SolrCore.<init> JMX monitoring not detected for core: collection1
   [junit4]   2> 188035 T340 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.schema.TestCloudManagedSchemaConcurrent 124C9216793140FD-001/control-001/cores/collection1/data
   [junit4]   2> 188036 T340 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.schema.TestCloudManagedSchemaConcurrent 124C9216793140FD-001/control-001/cores/collection1/data/index/
   [junit4]   2> 188037 T340 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.schema.TestCloudManagedSchemaConcurrent 124C9216793140FD-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 188038 T340 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.schema.TestCloudManagedSchemaConcurrent 124C9216793140FD-001/control-001/cores/collection1/data/index
   [junit4]   2> 188038 T340 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=41, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7434205957933164]
   [junit4]   2> 188045 T340 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=NRTCachingDirectory(MMapDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.schema.TestCloudManagedSchemaConcurrent 124C9216793140FD-001/control-001/cores/collection1/data/index lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@674358e3; maxCacheMB=48.0 maxMergeSizeMB=4.0),segFN=segments_1,generation=1}
   [junit4]   2> 188045 T340 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 188048 T340 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 188049 T340 C:control_collection c:collection1 oasc.RequestHandlers.applyInitParams WARN INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,args = {defaults={a=A,b=B}}}
   [junit4]   2> 188066 T340 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 188069 T340 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 188071 T340 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 188073 T340 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 188096 T340 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/file,/admin/mbeans,/get,/admin/logging,/admin/properties,standard,/dump,/admin/luke,/admin/threads,/admin/plugins,/admin/segments,/update,/admin/ping,/replication,/config,/schema,/update/json,/admin/system,/update/csv,/update/json/docs
   [junit4]   2> 188097 T340 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 188098 T340 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 188098 T340 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 188100 T340 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 188100 T340 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 188101 T340 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=15, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 188102 T340 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=NRTCachingDirectory(MMapDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.schema.TestCloudManagedSchemaConcurrent 124C9216793140FD-001/control-001/cores/collection1/data/index lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@674358e3; maxCacheMB=48.0 maxMergeSizeMB=4.0),segFN=segments_1,generation=1}
   [junit4]   2> 188103 T340 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 188103 T340 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@250a1b4d[collection1] main
   [junit4]   2> 188103 T340 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 188104 T340 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 188105 T340 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 188105 T340 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 188105 T340 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 188106 T340 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 188106 T340 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 188107 T340 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 188107 T340 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 188107 T340 C:control_collection c:collection1 oass.ZkIndexSchemaReader.createSchemaWatcher Creating ZooKeeper watch for the managed schema at /configs/conf1/managed-schema
   [junit4]   2> 188108 T340 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 188108 T341 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@250a1b4d[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 188109 T340 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 188110 T344 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:33185 collection:control_collection shard:shard1
   [junit4]   2> 188110 T310 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 188110 T310 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 188110 T344 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 188111 T310 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 188112 T310 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 188113 T347 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@186ad8c6 name:ZooKeeperConnection Watcher:127.0.0.1:33182/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 188113 T310 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 188114 T310 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 188114 T310 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 188116 T344 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 188117 T310 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 188118 T336 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 188118 T344 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 188119 T344 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C1643 name=collection1 org.apache.solr.core.SolrCore@5a4b4352 url=http://127.0.0.1:33185/collection1 node=127.0.0.1:33185_ C1643_STATE=coll:control_collection core:collection1 props:{core=collection1, node_name=127.0.0.1:33185_, base_url=http://127.0.0.1:33185, state=down}
   [junit4]   2> 188119 T344 C:control_collection S:shard1 C1643 P33185 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:33185/collection1/
   [junit4]   2> 188119 T337 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 188119 T344 C:control_collection S:shard1 C1643 P33185 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 188119 T344 C:control_collection S:shard1 C1643 P33185 oasc.SyncStrategy.syncToMe http://127.0.0.1:33185/collection1/ has no replicas
   [junit4]   2> 188120 T344 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:33185/collection1/ shard1
   [junit4]   2> 188120 T344 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 188121 T336 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 188121 T347 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 188125 T336 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 188126 T337 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33185",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 188229 T347 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 188229 T336 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 188278 T344 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:33185/collection1/ and leader is http://127.0.0.1:33185/collection1/
   [junit4]   2> 188279 T344 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:33185
   [junit4]   2> 188279 T344 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 188279 T344 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 188280 T344 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 188281 T336 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 188282 T337 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"4",
   [junit4]   2> 	  "node_name":"127.0.0.1:33185_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33185",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1"} current state version: 3
   [junit4]   2> 188283 T337 oasco.ReplicaMutator.updateState Update state numShards=4 message={
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"4",
   [junit4]   2> 	  "node_name":"127.0.0.1:33185_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33185",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1"}
   [junit4]   2> 188386 T347 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 188386 T336 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 189196 T310 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.schema.TestCloudManagedSchemaConcurrent 124C9216793140FD-001/shard-1-001/cores/collection1
   [junit4]   2> 189199 T310 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.schema.TestCloudManagedSchemaConcurrent 124C9216793140FD-001/shard-1-001
   [junit4]   2> 189200 T310 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 189203 T310 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:33193
   [junit4]   2> 189203 T310 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solrconfig=solrconfig.xml, hostContext=/, hostPort=33193, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.schema.TestCloudManagedSchemaConcurrent 124C9216793140FD-001/shard-1-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.schema.TestCloudManagedSchemaConcurrent 124C9216793140FD-001/tempDir-001/jetty1}
   [junit4]   2> 189204 T310 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 189204 T310 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.schema.TestCloudManagedSchemaConcurrent 124C9216793140FD-001/shard-1-001/'
   [junit4]   2> 189252 T310 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.schema.TestCloudManagedSchemaConcurrent 124C9216793140FD-001/shard-1-001/solr.xml
   [junit4]   2> 189309 T310 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.schema.TestCloudManagedSchemaConcurrent 124C9216793140FD-001/shard-1-001/cores
   [junit4]   2> 189310 T310 oasc.CoreContainer.<init> New CoreContainer 1872005178
   [junit4]   2> 189310 T310 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.schema.TestCloudManagedSchemaConcurrent 124C9216793140FD-001/shard-1-001/]
   [junit4]   2> 189310 T310 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.schema.TestCloudManagedSchemaConcurrent 124C9216793140FD-001/shard-1-001/lib
   [junit4]   2> 189312 T310 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/../../../../../../../../../../../usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.schema.TestCloudManagedSchemaConcurrent 124C9216793140FD-001/shard-1-001/lib).
   [junit4]   2> 189322 T310 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 189322 T310 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 189322 T310 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 189323 T310 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 189323 T310 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 189323 T310 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 189323 T310 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 189324 T310 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 189324 T310 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 189324 T310 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 189324 T310 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 189325 T310 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 189325 T310 oasl.LogWatcher.createWatcher SLF4J impl is org.

[...truncated too long message...]

jxx/lj,null}
   [junit4]   2> 1702658 T3355 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:32169 32169
   [junit4]   2> 1703727 T3356 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:32169 32169
   [junit4]   2> 1703728 T3356 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1703728 T3355 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:52690/kjxx/lj
   [junit4]   2> 1703729 T3355 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:13386/kjxx/lj
   [junit4]   2> 1703729 T3355 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:45819/kjxx/lj
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=124C9216793140FD -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=en_SG -Dtests.timezone=Europe/Stockholm -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   68.3s J0 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([124C9216793140FD:9A18ADCCD7CD2D05]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 1703739 T3355 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 124C9216793140FD-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {intDefault=PostingsFormat(name=Memory doPackFST= false), id=PostingsFormat(name=Direct), _version_=PostingsFormat(name=Memory doPackFST= false), range_facet_l=PostingsFormat(name=Asserting), multiDefault=PostingsFormat(name=Asserting), a_t=PostingsFormat(name=Asserting), timestamp=PostingsFormat(name=Asserting), text=FSTOrd50}, docValues:{timestamp=DocValuesFormat(name=Lucene50)}, sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {}, locale=en_SG, timezone=Europe/Stockholm
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=99091240,total=315621376
   [junit4]   2> NOTE: All tests run in this JVM: [TestStressReorder, DefaultValueUpdateProcessorTest, DistributedMLTComponentTest, CursorMarkTest, TestObjectReleaseTracker, SolrXmlInZkTest, TestClusterStateMutator, DocValuesMultiTest, CacheHeaderTest, LoggingHandlerTest, TestAnalyzedSuggestions, TestInitQParser, LeaderElectionIntegrationTest, TestCSVLoader, TestHdfsUpdateLog, TestUpdate, TestReRankQParserPlugin, HighlighterConfigTest, RollingRestartTest, CurrencyFieldOpenExchangeTest, TestMergePolicyConfig, TestDynamicFieldCollectionResource, HdfsUnloadDistributedZkTest, SpatialRPTFieldTypeTest, TestFastOutputStream, TestOverriddenPrefixQueryForCustomFieldType, OutputWriterTest, TestConfigOverlay, TestNRTOpen, BadCopyFieldTest, SpellCheckComponentTest, HighlighterMaxOffsetTest, TestRandomMergePolicy, TestSimpleTrackingShardHandler, TestShortCircuitedRequests, MBeansHandlerTest, WordBreakSolrSpellCheckerTest, QueryResultKeyTest, ChangedSchemaMergeTest, TestSimpleQParserPlugin, TestFastLRUCache, TestRebalanceLeaders, QueryElevationComponentTest, DistributedDebugComponentTest, URLClassifyProcessorTest, HdfsBasicDistributedZk2Test, SolrTestCaseJ4Test, TestLazyCores, ZkNodePropsTest, TestMiniSolrCloudCluster, TestDynamicFieldResource, BadComponentTest, StressHdfsTest, PolyFieldTest, DocValuesMissingTest, BinaryUpdateRequestHandlerTest, TestSchemaResource, HttpPartitionTest, BasicDistributedZkTest, LeaderInitiatedRecoveryOnCommitTest, TestBlobHandler, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J0 in 68.38s, 1 test, 1 error <<< FAILURES!

[...truncated 724 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 2 failures, 62 ignored (22 assumptions)

Total time: 60 minutes 30 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 217369615 bytes
Compression is 0.0%
Took 44 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2862 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2862/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:64776/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:64776/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([E3F18CA6622F31EC:6BA5B37CCCD35C14]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([E3F18CA6622F31EC:6BA5B37CCCD35C14]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([E3F18CA6622F31EC:6BA5B37CCCD35C14]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9322 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/init-core-data-001
   [junit4]   2> 378317 T750 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 378321 T750 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 378321 T751 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 378421 T750 oasc.ZkTestServer.run start zk server on port:48130
   [junit4]   2> 378422 T750 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 378422 T750 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 378426 T758 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6b7449d2 name:ZooKeeperConnection Watcher:127.0.0.1:48130 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 378426 T750 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 378427 T750 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 378427 T750 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 378429 T750 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 378430 T750 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 378431 T761 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2b052e3e name:ZooKeeperConnection Watcher:127.0.0.1:48130/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 378431 T750 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 378431 T750 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 378432 T750 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 378434 T750 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 378435 T750 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 378436 T750 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 378438 T750 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 378438 T750 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 378441 T750 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 378441 T750 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 378443 T750 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 378444 T750 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 378445 T750 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 378446 T750 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 378447 T750 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 378448 T750 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 378449 T750 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 378450 T750 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 378451 T750 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 378452 T750 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 378453 T750 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 378454 T750 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 378456 T750 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 378456 T750 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 378458 T750 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 378458 T750 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 378460 T750 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 378460 T750 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 379420 T750 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/control-001/cores/collection1
   [junit4]   2> 379423 T750 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 379426 T750 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:47777
   [junit4]   2> 379427 T750 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/control-001/cores, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/tempDir-001/control/data, hostPort=33179}
   [junit4]   2> 379427 T750 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 379427 T750 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/control-001/'
   [junit4]   2> 379457 T750 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/control-001/solr.xml
   [junit4]   2> 379492 T750 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/control-001/cores
   [junit4]   2> 379492 T750 oasc.CoreContainer.<init> New CoreContainer 362634595
   [junit4]   2> 379493 T750 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/control-001/]
   [junit4]   2> 379493 T750 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/control-001/lib
   [junit4]   2> 379493 T750 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/control-001/lib).
   [junit4]   2> 379502 T750 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 379503 T750 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 379503 T750 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 379503 T750 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 379503 T750 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 379503 T750 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 379504 T750 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 379504 T750 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 379504 T750 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 379504 T750 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 379504 T750 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 379505 T750 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 379505 T750 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 379505 T750 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 379505 T750 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 379506 T750 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:48130/solr
   [junit4]   2> 379506 T750 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 379506 T750 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 379507 T750 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 379508 T773 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@21dce344 name:ZooKeeperConnection Watcher:127.0.0.1:48130 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 379508 T750 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 379509 T750 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 379511 T750 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 379512 T776 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4b4468f6 name:ZooKeeperConnection Watcher:127.0.0.1:48130/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 379512 T750 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 379513 T750 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 379515 T750 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 379516 T750 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 379518 T750 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 379520 T750 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 379521 T750 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 379523 T750 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 379524 T750 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 379525 T750 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33179_
   [junit4]   2> 379526 T750 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:33179_
   [junit4]   2> 379527 T750 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 379528 T750 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 379530 T750 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 379531 T750 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:33179_
   [junit4]   2> 379531 T750 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 379532 T750 oasc.Overseer.start Overseer (id=93569762076721155-127.0.0.1:33179_-n_0000000000) starting
   [junit4]   2> 379534 T750 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 379539 T750 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 379540 T778 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 379540 T750 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 379540 T777 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 379543 T750 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/control-001/cores
   [junit4]   2> 379545 T750 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {dataDir=data/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/control-001/cores/collection1, config=solrconfig.xml, coreNodeName=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/control-001/cores/collection1/, name=collection1, collection=control_collection, loadOnStartup=true, shard=, schema=schema.xml, transient=false}
   [junit4]   2> 379545 T750 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/control-001/cores/collection1/
   [junit4]   2> 379545 T750 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 379546 T780 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 379546 T780 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 379547 T776 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 379547 T780 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 379548 T777 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33179",
   [junit4]   2> 	  "node_name":"127.0.0.1:33179_",
   [junit4]   2> 	  "operation":"state"} current state version: 0
   [junit4]   2> 379548 T777 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33179",
   [junit4]   2> 	  "node_name":"127.0.0.1:33179_",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 379548 T777 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 379548 T777 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 379549 T776 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 380547 T780 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 380548 T780 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 380548 T780 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 380549 T780 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 380549 T780 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 380550 T780 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/control-001/cores/collection1/'
   [junit4]   2> 380562 T780 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 380564 T780 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 380573 T780 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 380588 T780 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 380622 T780 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 380623 T780 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 380631 T780 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 380924 T780 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 380935 T780 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 380937 T780 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 380948 T780 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 380952 T780 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 380955 T780 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 380956 T780 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 380956 T780 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 380956 T780 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 380957 T780 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 380957 T780 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 380957 T780 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 380957 T780 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 380958 T780 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/control-001/cores/collection1/data/
   [junit4]   2> 380958 T780 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2afd506
   [junit4]   2> 380958 T780 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/control-001/cores/collection1/data
   [junit4]   2> 380959 T780 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/control-001/cores/collection1/data/index/
   [junit4]   2> 380959 T780 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 380959 T780 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/control-001/cores/collection1/data/index
   [junit4]   2> 380959 T780 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=37, maxMergeAtOnceExplicit=16, maxMergedSegmentMB=95.7392578125, floorSegmentMB=0.294921875, forceMergeDeletesPctAllowed=1.7430136491797477, segmentsPerTier=39.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6773007926290789
   [junit4]   2> 380960 T780 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3c43a547 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7d17f739),segFN=segments_1,generation=1}
   [junit4]   2> 380960 T780 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 380964 T780 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 380964 T780 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 380965 T780 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 380965 T780 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 380965 T780 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 380965 T780 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 380966 T780 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 380966 T780 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 380966 T780 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 380978 T780 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 380979 T780 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 380981 T780 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 380982 T780 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 380986 T780 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update,/schema,/admin/logging,/config,/admin/segments,/admin/threads,/admin/mbeans,/update/csv,/replication,/update/json,/update/json/docs,/get,/admin/luke,/admin/ping,/admin/file,/admin/properties,standard,/admin/system,/admin/plugins
   [junit4]   2> 380988 T780 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 380989 T780 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 380989 T780 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 380990 T780 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 380990 T780 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 380990 T780 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=2071242227, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 380991 T780 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3c43a547 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7d17f739),segFN=segments_1,generation=1}
   [junit4]   2> 380991 T780 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 380991 T780 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@79bcb231[collection1] main
   [junit4]   2> 380992 T780 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 380992 T780 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 380993 T780 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 380993 T780 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 380993 T780 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 380993 T780 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 380994 T780 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 380994 T780 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 380994 T780 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 380994 T780 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 380995 T781 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@79bcb231[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 380996 T780 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 380996 T784 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:33179 collection:control_collection shard:shard1
   [junit4]   2> 380997 T750 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 380997 T750 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 380998 T784 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 381000 T750 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 381001 T750 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 381002 T784 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 381003 T788 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2da88284 name:ZooKeeperConnection Watcher:127.0.0.1:48130/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 381003 T750 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 381003 T750 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 381003 T750 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 381004 T776 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 381004 T784 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 381005 T784 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C337 name=collection1 org.apache.solr.core.SolrCore@2166d861 url=http://127.0.0.1:33179/collection1 node=127.0.0.1:33179_ C337_STATE=coll:control_collection core:collection1 props:{core=collection1, state=down, base_url=http://127.0.0.1:33179, node_name=127.0.0.1:33179_}
   [junit4]   2> 381005 T784 C:control_collection S:shard1 C337 P33179 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:33179/collection1/
   [junit4]   2> 381005 T777 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 381005 T784 C:control_collection S:shard1 C337 P33179 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 381006 T784 C:control_collection S:shard1 C337 P33179 oasc.SyncStrategy.syncToMe http://127.0.0.1:33179/collection1/ has no replicas
   [junit4]   2> 381006 T784 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:33179/collection1/ shard1
   [junit4]   2> 381006 T784 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 381007 T750 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 381007 T750 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 381008 T750 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 381008 T750 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 381010 T791 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@19183043 name:ZooKeeperConnection Watcher:127.0.0.1:48130/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 381010 T750 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 381010 T750 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 381010 T776 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 381012 T777 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33179",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 1
   [junit4]   2> 381014 T777 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 1
   [junit4]   2> 381014 T777 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 381015 T788 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 381015 T776 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 381019 T776 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 381020 T777 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 381021 T776 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 381021 T788 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 381061 T784 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:33179/collection1/ and leader is http://127.0.0.1:33179/collection1/
   [junit4]   2> 381062 T784 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:33179
   [junit4]   2> 381062 T784 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 381062 T784 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 381062 T784 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 381063 T776 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 381064 T777 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33179",
   [junit4]   2> 	  "node_name":"127.0.0.1:33179_",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state"} current state version: 3
   [junit4]   2> 381065 T777 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33179",
   [junit4]   2> 	  "node_name":"127.0.0.1:33179_",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 381168 T788 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 381168 T776 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 382249 T750 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/shard-1-001/cores/collection1
   [junit4]   2> 382251 T750 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/shard-1-001
   [junit4]   2> 382252 T750 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 382254 T750 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:56358
   [junit4]   2> 382254 T750 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/shard-1-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=37798}
   [junit4]   2> 382254 T750 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 382255 T750 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/shard-1-001/'
   [junit4]   2> 382284 T750 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/shard-1-001/solr.xml
   [junit4]   2> 382319 T750 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/shard-1-001/cores
   [junit4]   2> 382319 T750 oasc.CoreContainer.<init> New CoreContainer 1656274388
   [junit4]   2> 382319 T750 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/shard-1-001/]
   [junit4]   2> 382319 T750 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/shard-1-001/lib
   [junit4]   2> 382320 T750 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/shard-1-001/lib).
   [junit4]   2> 382329 T750 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 382329 T750 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 382329 T750 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 382330 T750 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 382330 T750 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 382330 T750 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 382330 T750 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 382330 T750 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 382331 T750 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 382331 T750 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 382331 T750 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 382331 T750 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 382332 T750 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 382332 T750 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 382332 T750 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 382332 T750 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:48130/solr
   [junit4]   2> 382332 T750 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 382333 T750 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 382333 T750 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 382335 T803 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7860fa99 name:ZooKeeperConnection Watcher:127.0.0.1:48130 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 382335 T750 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 382336 T750 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 382337 T750 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 382338 T806 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4d9ab947 name:ZooKeeperConnection Watcher:127.0.0.1:48130/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 382339 T750 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 382343 T750 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 383348 T750 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37798_
   [junit4]   2> 383349 T750 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:37798_
   [junit4]   2> 383352 T750 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 383354 T750 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/shard-1-001/cores
   [junit4]   2> 383356 T750 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/shard-1-001/cores/collection1, collection=collection1, loadOnStartup=true, coreNodeName=, config=solrconfig.xml, transient=false, shard=, dataDir=data/, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/shard-1-001/cores/collection1/}
   [junit4]   2> 383356 T750 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/shard-1-001/cores/collection1/
   [junit4]   2> 383356 T750 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 383359 T807 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 383359 T807 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 383360 T776 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 383361 T807 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 383361 T777 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37798",
   [junit4]   2> 	  "node_name":"127.0.0.1:37798_",
   [junit4]   2> 	  "operation":"state"} current state version: 4
   [junit4]   2> 383361 T807 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 383362 T777 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37798",
   [junit4]   2> 	  "node_name":"127.0.0.1:37798_",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 383362 T807 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 383362 T777 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 383362 T777 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 383363 T807 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 383364 T807 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 383364 T777 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 383365 T806 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 383365 T806 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 384364 T807 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 384365 T807 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 384365 T807 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 384365 T807 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 384366 T807 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 384366 T807 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/shard-1-001/cores/collection1/'
   [junit4]   2> 384379 T807 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 384381 T807 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 384390 T807 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 384405 T807 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 384439 T807 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 384440 T807 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 384448 T807 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 384729 T807 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 384740 T807 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 384742 T807 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 384753 T807 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 384756 T807 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 384759 T807 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 384760 T807 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 384760 T807 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 384761 T807 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 384761 T807 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 384761 T807 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 384762 T807 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 384762 T807 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 384762 T807 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 384762 T807 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2afd506
   [junit4]   2> 384763 T807 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/shard-1-001/cores/collection1/data
   [junit4]   2> 384763 T807 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 384763 T807 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 384763 T807 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E3F18CA6622F31EC-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 384764 T807 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=37, maxMergeAtOnceExplicit=16, maxMergedSegmentMB=95.7392578125, floorSegmentMB=0.294921875, forceMergeDeletesPctAllowed=1.7430136491797477, segmentsPerTier=39.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6773007926290789
   [junit4]   2> 384764 T807 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2c59476b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@195fd88c),segFN=segments_1,generation=1}
   [junit4]   2> 384765 T807 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 384770 T807 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 384770 T807 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 384770 T807 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 384770 T807 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 384770 T807 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 384771 T807 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 384771 T807 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 384771 T807 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 384771 T807 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 384783 T807 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 384785 T807 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 384786 T807 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 384788 T807 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 384793 T807 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update,/schema,/admin/logging,/config,/admin/segments,/admin/threads,/admin/mbeans,/update/csv,/replication,/update/json,/update/json/docs,/get,/admin/luke,/admin/ping,/admin/file,/admin/properties,standard,/admin/system,/admin/plugins
   [junit4]   2> 384794 T807 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 384795 T807 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 384796 T807 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 384797 T807 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 384797 T807 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 384798 T807 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=2071242227, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 384798 T807 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2c59476b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@195fd88c),segFN=segments_1,generation=1}
   [junit4]   2> 384798 T807 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 384799 T807 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@2d0a10bd[collection1] main
   [junit4]   2> 384799 T807 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 384800 T807 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 384800 T807 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 384800 T807 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 384800 T807 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 384800 T807 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 384801 T807 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 384801 T807 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 384801 T807 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 384802 T807 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 384802 T808 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2d0a10bd[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 384803 T807 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 384804 T811 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:37798 collection:collection1 shard:shard1
   [junit4]   2> 384804 T750 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 384805 T750 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 384805 T811 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 384809 T811 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 384811 T776 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 384811 T811 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 384811 T811 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C338 name=collection1 org.apache.solr.core.SolrCore@1b7a1fb0 url=http://127.0.0.1:37798/collection1 node=127.0.0.1:37798_ C338_STATE=coll:collection1 core:collection1 props:{core=collection1, state=down, base_url=http://127.0.0.1:37798, node_name=127.0.0.1:37798_}
   [junit4]   2> 384812 T811 C:collection1 S:shard1 C338 P37798 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:37798/collection1/
   [junit4]   2> 384812 T777 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 4
   [junit4]   2> 384812 T811 C:collection1 S:shard1 C338 P37798 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 384812 T811 C:collection1 S:shard1 C338 P37798 oasc.SyncStrategy.syncToMe http://127.0.0.1:37798/collection1/ has no replicas
   [junit4]   2> 384812 T811 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:37798/collection1/ shard1
   [junit4]   2> 384813 T811 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 384813 T777 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 384814 T806 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 384814 T806 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 384817 T777 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37798",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 4
   [junit4]   2> 384818 T777 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 384819 T806 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 384819 T806 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 384821 T776 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 384867 T811 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:37798/collection1/ and leader is http://127.0.0.1:37798/collection1/
   [junit4]   2> 384868 T811 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:37798
   [junit4]   2> 384868 T811 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 384868 T811 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 384868 T811 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 384869 T776 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 384870 T777 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37798",
   [junit4]   2> 	  "node_name":"127.0.0.1:37798_",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state"} current state version: 4
   [junit4]   2> 384871 T777 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  

[...truncated too long message...]

88 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 10 connections to: http://127.0.0.1:63464/
   [junit4]   2> 1572506 T5888 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:63469/
   [junit4]   2> 1572506 T5888 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 1 connections to: http://127.0.0.1:63446/
   [junit4]   2> 1572506 T5888 C:control_collection S:shard1 c:collection1 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 1572507 T5888 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:63458/
   [junit4]   2> 1572507 T5888 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:63464/
   [junit4]   2> 1572507 T5888 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:63469/
   [junit4]   2> 1572508 T5888 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:63446/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=E3F18CA6622F31EC -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=fi -Dtests.timezone=America/Indianapolis -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   76.4s J2 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([E3F18CA6622F31EC:6BA5B37CCCD35C14]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 1572531 T5888 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest E3F18CA6622F31EC-001
   [junit4]   2> 76402 T5887 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {range_facet_l=PostingsFormat(name=Asserting), timestamp=PostingsFormat(name=Asserting), multiDefault=PostingsFormat(name=Asserting), a_t=PostingsFormat(name=Asserting), text=Lucene50(blocksize=128), id=PostingsFormat(name=MockRandom), _version_=PostingsFormat(name=LuceneVarGapFixedInterval), intDefault=PostingsFormat(name=LuceneVarGapFixedInterval)}, docValues:{timestamp=DocValuesFormat(name=Memory)}, sim=DefaultSimilarity, locale=fi, timezone=America/Indianapolis
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=149301048,total=434110464
   [junit4]   2> NOTE: All tests run in this JVM: [CSVRequestHandlerTest, TestCSVLoader, SuggesterTSTTest, TermVectorComponentTest, TestRandomFaceting, SolrIndexSplitterTest, BlockCacheTest, OverseerCollectionProcessorTest, TestRandomMergePolicy, AliasIntegrationTest, TestReqParamsAPI, TestDocBasedVersionConstraints, TestQuerySenderNoQuery, ConvertedLegacyTest, TestAtomicUpdateErrorCases, TestSurroundQueryParser, TestAddFieldRealTimeGet, DocumentBuilderTest, SearchHandlerTest, QueryResultKeyTest, TestFieldTypeResource, TestPartialUpdateDeduplication, AnalyticsQueryTest, SyncSliceTest, TestSystemIdResolver, HdfsRecoveryZkTest, TestMaxScoreQueryParser, TestClusterStateMutator, ShardRoutingTest, SignatureUpdateProcessorFactoryTest, TestFieldResource, TestFieldCollectionResource, DistanceFunctionTest, TestSolrDynamicMBean, TestRestManager, TestQueryTypes, TestUniqueKeyFieldResource, TestBulkSchemaConcurrent, SoftAutoCommitTest, TestDynamicFieldCollectionResource, SpellCheckComponentTest, SharedFSAutoReplicaFailoverTest, ConnectionManagerTest, TestPseudoReturnFields, TestCoreDiscovery, TestOrdValues, TestBulkSchemaAPI, TestManagedSchema, DebugComponentTest, ZkNodePropsTest, TestHdfsUpdateLog, TestIndexSearcher, RecoveryZkTest, LeaderElectionIntegrationTest, SOLR749Test, TestCollationFieldDocValues, TestLRUStatsCache, TestSweetSpotSimilarityFactory, TestStandardQParsers, OpenExchangeRatesOrgProviderTest, TestCloudManagedSchemaConcurrent, CollectionsAPIDistributedZkTest, TestCloudSchemaless, CustomCollectionTest, MultiThreadedOCPTest, ReplicationFactorTest]
   [junit4] Completed on J2 in 77.19s, 1 test, 1 error <<< FAILURES!

[...truncated 716 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 62 ignored (22 assumptions)

Total time: 54 minutes 11 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 239413095 bytes
Compression is 0.0%
Took 44 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2861 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2861/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:20392/t/qu/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:20392/t/qu/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([FC65B5FF1C8C2C10:74318A25B27041E8]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([FC65B5FF1C8C2C10:74318A25B27041E8]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([FC65B5FF1C8C2C10:74318A25B27041E8]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9348 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/init-core-data-001
   [junit4]   2> 227457 T987 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /t/qu
   [junit4]   2> 227462 T987 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 227463 T988 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 227562 T987 oasc.ZkTestServer.run start zk server on port:20387
   [junit4]   2> 227563 T987 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 227564 T987 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 227567 T995 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2845a27c name:ZooKeeperConnection Watcher:127.0.0.1:20387 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 227567 T987 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 227567 T987 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 227567 T987 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 227569 T987 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 227570 T987 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 227571 T998 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@47d9b85b name:ZooKeeperConnection Watcher:127.0.0.1:20387/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 227571 T987 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 227572 T987 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 227572 T987 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 227573 T987 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 227574 T987 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 227575 T987 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 227576 T987 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 227577 T987 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 227579 T987 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 227579 T987 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 227581 T987 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 227581 T987 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 227582 T987 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 227583 T987 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 227584 T987 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 227584 T987 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 227586 T987 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 227586 T987 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 227587 T987 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 227588 T987 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 227589 T987 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 227590 T987 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 227591 T987 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 227591 T987 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 227593 T987 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 227593 T987 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 227595 T987 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 227595 T987 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 228561 T987 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/control-001/cores/collection1
   [junit4]   2> 228564 T987 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 228568 T987 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:20393
   [junit4]   2> 228568 T987 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/tempDir-001/control/data, hostContext=/t/qu, hostPort=20392}
   [junit4]   2> 228568 T987 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 228569 T987 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/control-001/'
   [junit4]   2> 228598 T987 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/control-001/solr.xml
   [junit4]   2> 228641 T987 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/control-001/cores
   [junit4]   2> 228641 T987 oasc.CoreContainer.<init> New CoreContainer 1118964066
   [junit4]   2> 228642 T987 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/control-001/]
   [junit4]   2> 228642 T987 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/control-001/lib
   [junit4]   2> 228642 T987 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/control-001/lib).
   [junit4]   2> 228651 T987 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 228652 T987 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 228652 T987 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 228652 T987 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 228652 T987 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 228653 T987 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 228653 T987 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 228653 T987 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 228653 T987 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 228653 T987 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 228654 T987 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 228654 T987 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 228654 T987 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 228654 T987 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 228655 T987 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 228655 T987 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:20387/solr
   [junit4]   2> 228655 T987 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 228655 T987 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 228656 T987 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 228658 T1010 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@325ace80 name:ZooKeeperConnection Watcher:127.0.0.1:20387 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 228658 T987 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 228659 T987 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 228660 T987 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 228661 T1013 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@428f7692 name:ZooKeeperConnection Watcher:127.0.0.1:20387/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 228661 T987 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 228662 T987 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 228663 T987 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 228665 T987 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 228666 T987 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 228667 T987 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 228669 T987 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 228670 T987 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 228671 T987 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 228671 T987 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:20392_t%2Fqu
   [junit4]   2> 228672 T987 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:20392_t%2Fqu
   [junit4]   2> 228673 T987 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 228674 T987 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 228675 T987 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 228676 T987 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:20392_t%2Fqu
   [junit4]   2> 228676 T987 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 228677 T987 oasc.Overseer.start Overseer (id=93568810239655939-127.0.0.1:20392_t%2Fqu-n_0000000000) starting
   [junit4]   2> 228678 T987 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 228682 T987 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 228683 T1015 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 228683 T1014 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 228683 T987 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 228686 T987 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/control-001/cores
   [junit4]   2> 228688 T987 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, schema=schema.xml, collection=control_collection, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/control-001/cores/collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/control-001/cores/collection1/, transient=false, coreNodeName=, shard=, loadOnStartup=true, dataDir=data/, config=solrconfig.xml}
   [junit4]   2> 228688 T987 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/control-001/cores/collection1/
   [junit4]   2> 228688 T987 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 228689 T1017 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 228689 T1017 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 228690 T1017 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 228690 T1013 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 228691 T1014 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20392/t/qu",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:20392_t%2Fqu",
   [junit4]   2> 	  "core":"collection1"} current state version: 0
   [junit4]   2> 228691 T1014 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20392/t/qu",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:20392_t%2Fqu",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 228691 T1014 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 228692 T1014 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 228692 T1013 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 229690 T1017 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 229691 T1017 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 229691 T1017 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 229692 T1017 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 229692 T1017 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 229692 T1017 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/control-001/cores/collection1/'
   [junit4]   2> 229710 T1017 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 229712 T1017 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 229723 T1017 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 229745 T1017 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 229793 T1017 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 229794 T1017 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 229806 T1017 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 230195 T1017 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 230212 T1017 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 230215 T1017 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 230226 T1017 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 230231 T1017 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 230235 T1017 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 230236 T1017 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 230236 T1017 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 230236 T1017 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 230237 T1017 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 230237 T1017 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 230237 T1017 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 230238 T1017 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 230238 T1017 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/control-001/cores/collection1/data/
   [junit4]   2> 230238 T1017 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3434f8f3
   [junit4]   2> 230239 T1017 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/control-001/cores/collection1/data
   [junit4]   2> 230239 T1017 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/control-001/cores/collection1/data/index/
   [junit4]   2> 230240 T1017 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 230240 T1017 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/control-001/cores/collection1/data/index
   [junit4]   2> 230241 T1017 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=31, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 230241 T1017 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1514360b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@61e816fc),segFN=segments_1,generation=1}
   [junit4]   2> 230242 T1017 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 230248 T1017 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 230249 T1017 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 230249 T1017 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 230249 T1017 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 230250 T1017 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 230250 T1017 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 230250 T1017 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 230251 T1017 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 230251 T1017 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 230270 T1017 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 230273 T1017 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 230275 T1017 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 230277 T1017 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 230283 T1017 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/threads,/update/json/docs,/admin/system,/replication,/admin/ping,/update/csv,/admin/luke,/update/json,/admin/logging,standard,/config,/get,/schema,/admin/segments,/admin/file,/admin/properties,/admin/mbeans,/admin/plugins,/update
   [junit4]   2> 230285 T1017 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 230287 T1017 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 230287 T1017 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 230289 T1017 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 230289 T1017 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 230290 T1017 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=18, maxMergeAtOnceExplicit=48, maxMergedSegmentMB=20.37109375, floorSegmentMB=0.3916015625, forceMergeDeletesPctAllowed=28.308577331717743, segmentsPerTier=40.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.23320560394696815
   [junit4]   2> 230290 T1017 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1514360b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@61e816fc),segFN=segments_1,generation=1}
   [junit4]   2> 230291 T1017 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 230291 T1017 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@18d907c[collection1] main
   [junit4]   2> 230291 T1017 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 230292 T1017 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 230292 T1017 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 230293 T1017 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 230293 T1017 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 230293 T1017 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 230293 T1017 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 230294 T1017 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 230294 T1017 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 230294 T1017 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 230295 T1018 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@18d907c[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 230296 T1017 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 230296 T1021 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:20392/t/qu collection:control_collection shard:shard1
   [junit4]   2> 230297 T987 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 230297 T987 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 230297 T1021 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 230300 T987 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 230300 T987 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 230301 T1021 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 230301 T1025 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7be1b35c name:ZooKeeperConnection Watcher:127.0.0.1:20387/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 230302 T987 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 230302 T1013 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 230302 T1021 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 230302 T1021 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 230302 T987 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 230303 T1014 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C100 name=collection1 org.apache.solr.core.SolrCore@42b57192 url=http://127.0.0.1:20392/t/qu/collection1 node=127.0.0.1:20392_t%2Fqu C100_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:20392/t/qu, state=down, node_name=127.0.0.1:20392_t%2Fqu, core=collection1}
   [junit4]   2> 230303 T1021 C:control_collection S:shard1 C100 P20392 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:20392/t/qu/collection1/
   [junit4]   2> 230303 T987 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 230303 T1021 C:control_collection S:shard1 C100 P20392 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 230304 T1021 C:control_collection S:shard1 C100 P20392 oasc.SyncStrategy.syncToMe http://127.0.0.1:20392/t/qu/collection1/ has no replicas
   [junit4]   2> 230304 T1021 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:20392/t/qu/collection1/ shard1
   [junit4]   2> 230304 T1021 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 230304 T1013 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 230304 T1025 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 230306 T987 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 230307 T1013 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 230308 T1014 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20392/t/qu",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 230411 T1025 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 230411 T1013 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 230460 T1021 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:20392/t/qu/collection1/ and leader is http://127.0.0.1:20392/t/qu/collection1/
   [junit4]   2> 230461 T1021 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:20392/t/qu
   [junit4]   2> 230461 T1021 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 230461 T1021 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 230461 T1021 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 230462 T1013 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 230463 T1014 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20392/t/qu",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:20392_t%2Fqu",
   [junit4]   2> 	  "core":"collection1"} current state version: 3
   [junit4]   2> 230463 T1014 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20392/t/qu",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:20392_t%2Fqu",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 230566 T1025 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 230566 T1013 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 231488 T987 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/shard-1-001/cores/collection1
   [junit4]   2> 231490 T987 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/shard-1-001
   [junit4]   2> 231491 T987 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 231493 T987 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:20401
   [junit4]   2> 231493 T987 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/t/qu, solrconfig=solrconfig.xml, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/shard-1-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/tempDir-001/jetty1, hostPort=20400}
   [junit4]   2> 231494 T987 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 231494 T987 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/shard-1-001/'
   [junit4]   2> 231523 T987 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/shard-1-001/solr.xml
   [junit4]   2> 231558 T987 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/shard-1-001/cores
   [junit4]   2> 231558 T987 oasc.CoreContainer.<init> New CoreContainer 848433980
   [junit4]   2> 231558 T987 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/shard-1-001/]
   [junit4]   2> 231559 T987 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/shard-1-001/lib
   [junit4]   2> 231559 T987 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/shard-1-001/lib).
   [junit4]   2> 231568 T987 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 231569 T987 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 231569 T987 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 231569 T987 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 231569 T987 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 231570 T987 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 231570 T987 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 231570 T987 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 231570 T987 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 231570 T987 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 231571 T987 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 231571 T987 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 231571 T987 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 231572 T987 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 231572 T987 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 231572 T987 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:20387/solr
   [junit4]   2> 231572 T987 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 231572 T987 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 231573 T987 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 231574 T1037 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1da50aba name:ZooKeeperConnection Watcher:127.0.0.1:20387 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 231575 T987 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 231576 T987 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 231577 T987 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 231578 T1040 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4b5ef4b7 name:ZooKeeperConnection Watcher:127.0.0.1:20387/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 231578 T987 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 231581 T987 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 232584 T987 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:20400_t%2Fqu
   [junit4]   2> 232585 T987 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:20400_t%2Fqu
   [junit4]   2> 232588 T987 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 232590 T987 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/shard-1-001/cores
   [junit4]   2> 232592 T987 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {shard=, name=collection1, collection=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/shard-1-001/cores/collection1/, coreNodeName=, dataDir=data/, loadOnStartup=true, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/shard-1-001/cores/collection1, transient=false, config=solrconfig.xml}
   [junit4]   2> 232592 T987 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/shard-1-001/cores/collection1/
   [junit4]   2> 232593 T987 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 232594 T1041 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 232595 T1041 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 232596 T1041 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 232596 T1013 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 232596 T1014 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20400/t/qu",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:20400_t%2Fqu",
   [junit4]   2> 	  "core":"collection1"} current state version: 4
   [junit4]   2> 232597 T1014 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20400/t/qu",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:20400_t%2Fqu",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 232597 T1014 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 232597 T1014 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 232599 T1013 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 232599 T1025 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 232599 T1040 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 233596 T1041 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 233597 T1041 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 233597 T1041 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 233597 T1041 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 233598 T1041 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 233598 T1041 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/shard-1-001/cores/collection1/'
   [junit4]   2> 233611 T1041 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 233613 T1041 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 233620 T1041 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 233635 T1041 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 233667 T1041 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 233668 T1041 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 233676 T1041 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 234051 T1041 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 234068 T1041 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 234071 T1041 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 234082 T1041 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 234087 T1041 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 234091 T1041 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 234092 T1041 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 234092 T1041 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 234092 T1041 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 234093 T1041 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 234093 T1041 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 234093 T1041 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 234094 T1041 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 234094 T1041 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 234094 T1041 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3434f8f3
   [junit4]   2> 234095 T1041 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/shard-1-001/cores/collection1/data
   [junit4]   2> 234096 T1041 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 234096 T1041 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 234096 T1041 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC65B5FF1C8C2C10-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 234097 T1041 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=31, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 234098 T1041 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@34270e4d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@701bfbfa),segFN=segments_1,generation=1}
   [junit4]   2> 234098 T1041 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 234106 T1041 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 234106 T1041 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 234107 T1041 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 234107 T1041 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 234107 T1041 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 234108 T1041 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 234108 T1041 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 234109 T1041 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 234109 T1041 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 234128 T1041 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 234131 T1041 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 234133 T1041 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 234136 T1041 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 234144 T1041 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/threads,/update/json/docs,/admin/system,/replication,/admin/ping,/update/csv,/admin/luke,/update/json,/admin/logging,standard,/config,/get,/schema,/admin/segments,/admin/file,/admin/properties,/admin/mbeans,/admin/plugins,/update
   [junit4]   2> 234147 T1041 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 234148 T1041 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 234148 T1041 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 234150 T1041 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 234150 T1041 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 234151 T1041 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=18, maxMergeAtOnceExplicit=48, maxMergedSegmentMB=20.37109375, floorSegmentMB=0.3916015625, forceMergeDeletesPctAllowed=28.308577331717743, segmentsPerTier=40.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.23320560394696815
   [junit4]   2> 234152 T1041 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@34270e4d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@701bfbfa),segFN=segments_1,generation=1}
   [junit4]   2> 234152 T1041 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 234152 T1041 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@4a98062b[collection1] main
   [junit4]   2> 234153 T1041 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 234153 T1041 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 234154 T1041 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 234154 T1041 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 234154 T1041 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 234155 T1041 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 234155 T1041 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 234155 T1041 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 234156 T1041 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 234156 T1041 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 234157 T1042 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4a98062b[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 234157 T1041 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 234158 T1045 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:20400/t/qu collection:collection1 shard:shard1
   [junit4]   2> 234158 T987 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 234159 T987 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 234159 T1045 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 234163 T1045 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 234164 T1013 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 234164 T1045 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 234164 T1045 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C101 name=collection1 org.apache.solr.core.SolrCore@5816a9bf url=http://127.0.0.1:20400/t/qu/collection1 node=127.0.0.1:20400_t%2Fqu C101_STATE=coll:collection1 core:collection1 props:{base_url=http://127.0.0.1:20400/t/qu, state=down, node_name=127.0.0.1:20400_t%2Fqu, core=collection1}
   [junit4]   2> 234164 T1045 C:collection1 S:shard1 C101 P20400 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:20400/t/qu/collection1/
   [junit4]   2> 234165 T1014 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 234165 T1045 C:collection1 S:shard1 C101 P20400 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 234165 T1045 C:collection1 S:shard1 C101 P20400 oasc.SyncStrategy.syncToMe http://127.0.0.1:20400/t/qu/collection1/ has no replicas
   [junit4]   2> 234165 T1045 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:20400/t/qu/collection1/ shard1
   [junit4]   2> 234165 T1045 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 234166 T1013 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 234166 T1025 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 234166 T1040 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 234169 T1014 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20400/t/qu",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 234171 T1013 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 234171 T1025 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 234171 T1040 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 234171 T1013 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 234219 T1045 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:20400/t/qu/collection1/ and leader is http://127.0.0.1:20400/t/qu/collection1/
   [junit4]   2> 234220 T1045 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:20400/t/qu
   [junit4]   2> 234220 T1045 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 234220 T1045 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 234220 T1045 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 234221 T1013 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 234222 T1014 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20400/t/qu",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:20400_t%2Fqu",
   [junit4]   2> 	  "core":"collection1"} current state version: 7
   [junit4]   2> 234223 T1014 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20400/t/qu",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:20400_t%2Fqu",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 234325 T1025 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 234325 T1013 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live node

[...truncated too long message...]

.0.0.1:20582 20582
   [junit4]   2> 1513836 T6455 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:20582 20582
   [junit4]   2> 1513837 T6455 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1513837 T6454 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 6 connections to: http://127.0.0.1:24287/
   [junit4]   2> 1513838 T6454 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:58475/
   [junit4]   2> 1513838 T6454 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:62071/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=FC65B5FF1C8C2C10 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=zh_SG -Dtests.timezone=America/North_Dakota/Center -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   72.0s J2 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([FC65B5FF1C8C2C10:74318A25B27041E8]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 1513855 T6454 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest FC65B5FF1C8C2C10-001
   [junit4]   2> 71980 T6453 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {range_facet_l=PostingsFormat(name=Direct), id=PostingsFormat(name=LuceneVarGapDocFreqInterval), _version_=FSTOrd50, timestamp=PostingsFormat(name=Direct), multiDefault=PostingsFormat(name=Direct), text=PostingsFormat(name=LuceneFixedGap), a_t=PostingsFormat(name=Direct), intDefault=FSTOrd50}, docValues:{timestamp=DocValuesFormat(name=Lucene50)}, sim=RandomSimilarityProvider(queryNorm=true,coord=no): {}, locale=zh_SG, timezone=America/North_Dakota/Center
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=110855848,total=358088704
   [junit4]   2> NOTE: All tests run in this JVM: [SpatialFilterTest, TestOverriddenPrefixQueryForCustomFieldType, ResponseLogComponentTest, TestRealTimeGet, ZkNodePropsTest, TestReplicationHandler, TestSolrQueryParserDefaultOperatorResource, TestRequestStatusCollectionAPI, TestBlendedInfixSuggestions, CollectionsAPIDistributedZkTest, OverseerTest, MultiTermTest, TestExactSharedStatsCache, AsyncMigrateRouteKeyTest, TestTrie, TestSolrQueryParser, SolrTestCaseJ4Test, TestSchemaResource, CoreAdminHandlerTest, TestCloudPivotFacet, TestIBSimilarityFactory, DistributedFacetPivotSmallTest, TestSchemaSimilarityResource, TestWordDelimiterFilterFactory, XmlUpdateRequestHandlerTest, TestCodecSupport, DistributedSuggestComponentTest, AutoCommitTest, FieldMutatingUpdateProcessorTest, SolrInfoMBeanTest, TestFastOutputStream, TestExpandComponent, TestDynamicFieldCollectionResource, AnalysisAfterCoreReloadTest, TestCollationFieldDocValues, SpellCheckCollatorTest, TestJmxMonitoredMap, SuggesterFSTTest, CSVRequestHandlerTest, TestSort, TestSolrConfigHandler, DocumentBuilderTest, SystemInfoHandlerTest, HdfsUnloadDistributedZkTest, TestFastWriter, TestManagedResource, TestDistributedMissingSort, DirectUpdateHandlerTest, TestCopyFieldCollectionResource, DistributedDebugComponentTest, SuggesterTSTTest, DateMathParserTest, TestNamedUpdateProcessors, ChaosMonkeyNothingIsSafeTest, ResponseHeaderTest, HdfsLockFactoryTest, TestCloudManagedSchemaConcurrent, FullSolrCloudDistribCmdsTest, DistribDocExpirationUpdateProcessorTest, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J2 in 72.76s, 1 test, 1 error <<< FAILURES!

[...truncated 714 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 62 ignored (22 assumptions)

Total time: 52 minutes 39 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 225566929 bytes
Compression is 0.0%
Took 48 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2860 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2860/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:51335/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:51335/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([FC97E93C8C0589AB:74C3D6E622F9E453]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:40258/i_bl/sd/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:40258/i_bl/sd/collection1
	at __randomizedtesting.SeedInfo.seed([FC97E93C8C0589AB:74C3D6E622F9E453]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:40258/i_bl/sd/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:51688/repfacttest_c8n_1x3_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:51688/repfacttest_c8n_1x3_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([FC97E93C8C0589AB:74C3D6E622F9E453]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:51688/repfacttest_c8n_1x3_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9423 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/init-core-data-001
   [junit4]   2> 189997 T666 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 190003 T666 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 190004 T667 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 190103 T666 oasc.ZkTestServer.run start zk server on port:51328
   [junit4]   2> 190104 T666 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 190105 T666 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 190109 T674 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@597afe97 name:ZooKeeperConnection Watcher:127.0.0.1:51328 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 190109 T666 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 190110 T666 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 190110 T666 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 190112 T666 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 190113 T666 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 190114 T677 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@51bf390b name:ZooKeeperConnection Watcher:127.0.0.1:51328/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 190115 T666 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 190115 T666 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 190115 T666 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 190117 T666 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 190118 T666 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 190119 T666 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 190121 T666 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 190121 T666 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 190123 T666 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 190124 T666 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 190226 T666 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 190226 T666 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 190228 T666 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 190229 T666 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 190230 T666 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 190231 T666 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 190232 T666 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 190232 T666 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 190234 T666 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 190234 T666 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 190236 T666 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 190236 T666 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 190238 T666 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 190238 T666 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 190240 T666 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 190240 T666 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 190242 T666 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 190242 T666 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 191786 T666 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/control-001/cores/collection1
   [junit4]   2> 191790 T666 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 191798 T666 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:51336
   [junit4]   2> 191799 T666 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=51335, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/control-001/cores, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/tempDir-001/control/data}
   [junit4]   2> 191799 T666 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 191800 T666 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/control-001/'
   [junit4]   2> 191837 T666 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/control-001/solr.xml
   [junit4]   2> 191885 T666 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/control-001/cores
   [junit4]   2> 191886 T666 oasc.CoreContainer.<init> New CoreContainer 164739511
   [junit4]   2> 191886 T666 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/control-001/]
   [junit4]   2> 191887 T666 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/control-001/lib
   [junit4]   2> 191887 T666 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/control-001/lib).
   [junit4]   2> 191897 T666 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 191897 T666 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 191898 T666 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 191898 T666 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 191898 T666 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 191899 T666 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 191899 T666 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 191899 T666 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 191899 T666 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 191900 T666 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 191900 T666 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 191900 T666 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 191901 T666 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 191901 T666 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 191901 T666 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 191902 T666 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51328/solr
   [junit4]   2> 191902 T666 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 191902 T666 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 191903 T666 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 191904 T689 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4b99ce2e name:ZooKeeperConnection Watcher:127.0.0.1:51328 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 191906 T666 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 191906 T666 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 191908 T666 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 191909 T692 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@72e78b2 name:ZooKeeperConnection Watcher:127.0.0.1:51328/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 191909 T666 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 191910 T666 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 191912 T666 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 191913 T666 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 191915 T666 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 191917 T666 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 191918 T666 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 191919 T666 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 191920 T666 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 191921 T666 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51335_
   [junit4]   2> 191922 T666 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:51335_
   [junit4]   2> 191923 T666 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 191924 T666 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 191925 T666 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 191926 T666 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:51335_
   [junit4]   2> 191927 T666 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 191928 T666 oasc.Overseer.start Overseer (id=93568035854155779-127.0.0.1:51335_-n_0000000000) starting
   [junit4]   2> 191929 T666 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 191934 T666 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 191934 T694 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 191935 T693 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 191934 T666 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 191938 T666 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/control-001/cores
   [junit4]   2> 191940 T666 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/control-001/cores/collection1, shard=, name=collection1, coreNodeName=, loadOnStartup=true, collection=control_collection, transient=false, config=solrconfig.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/control-001/cores/collection1/, dataDir=data/, schema=schema.xml}
   [junit4]   2> 191941 T666 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/control-001/cores/collection1/
   [junit4]   2> 191941 T666 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 191942 T696 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 191943 T696 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 191944 T696 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 191944 T692 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 191945 T693 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:51335_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51335",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 0
   [junit4]   2> 191945 T693 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:51335_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51335",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection"}
   [junit4]   2> 191945 T693 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 191946 T693 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 191947 T692 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 192944 T696 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 192945 T696 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 192946 T696 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 192946 T696 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 192946 T696 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 192947 T696 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/control-001/cores/collection1/'
   [junit4]   2> 192960 T696 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 192962 T696 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 192970 T696 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 192985 T696 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 193048 T696 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 193150 T696 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 193163 T696 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 193589 T696 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 193607 T696 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 193611 T696 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 193626 T696 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 193631 T696 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 193635 T696 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 193636 T696 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 193636 T696 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 193637 T696 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 193638 T696 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 193638 T696 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 193638 T696 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 193639 T696 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 193639 T696 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/control-001/cores/collection1/data/
   [junit4]   2> 193639 T696 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6e7ce201
   [junit4]   2> 193641 T696 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/control-001/cores/collection1/data
   [junit4]   2> 193641 T696 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/control-001/cores/collection1/data/index/
   [junit4]   2> 193641 T696 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 193642 T696 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/control-001/cores/collection1/data/index
   [junit4]   2> 193642 T696 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=14, maxMergeAtOnceExplicit=25, maxMergedSegmentMB=5.734375, floorSegmentMB=0.28125, forceMergeDeletesPctAllowed=2.459418236654205, segmentsPerTier=42.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7603132523207807
   [junit4]   2> 193643 T696 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4a08254d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5cc90833),segFN=segments_1,generation=1}
   [junit4]   2> 193644 T696 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 193652 T696 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 193652 T696 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 193653 T696 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 193653 T696 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 193653 T696 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 193654 T696 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 193654 T696 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 193655 T696 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 193655 T696 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 193677 T696 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 193679 T696 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 193682 T696 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 193684 T696 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 193691 T696 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update/csv,standard,/replication,/admin/plugins,/update/json,/update,/admin/system,/admin/logging,/admin/ping,/update/json/docs,/admin/properties,/get,/admin/mbeans,/admin/file,/config,/admin/threads,/admin/segments,/admin/luke,/schema
   [junit4]   2> 193693 T696 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 193695 T696 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 193695 T696 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 193697 T696 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 193697 T696 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 193698 T696 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=37, maxMergeAtOnceExplicit=14, maxMergedSegmentMB=6.4130859375, floorSegmentMB=0.2431640625, forceMergeDeletesPctAllowed=21.323455194532404, segmentsPerTier=14.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.42610527605398063
   [junit4]   2> 193699 T696 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4a08254d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5cc90833),segFN=segments_1,generation=1}
   [junit4]   2> 193699 T696 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 193700 T696 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@3af8000b[collection1] main
   [junit4]   2> 193700 T696 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 193701 T696 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 193701 T696 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 193702 T696 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 193702 T696 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 193703 T696 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 193703 T696 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 193704 T696 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 193704 T696 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 193704 T696 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 193705 T697 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3af8000b[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 193706 T696 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 193707 T700 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:51335 collection:control_collection shard:shard1
   [junit4]   2> 193707 T666 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 193708 T666 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 193708 T700 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 193711 T666 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 193711 T666 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 193712 T700 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 193713 T704 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1ec2a5a5 name:ZooKeeperConnection Watcher:127.0.0.1:51328/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 193713 T666 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 193714 T692 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 193714 T666 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 193714 T700 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 193714 T666 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 193714 T693 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 193714 T700 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C54 name=collection1 org.apache.solr.core.SolrCore@40b4efdb url=http://127.0.0.1:51335/collection1 node=127.0.0.1:51335_ C54_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:51335_, state=down, core=collection1, base_url=http://127.0.0.1:51335}
   [junit4]   2> 193715 T700 C:control_collection S:shard1 C54 P51335 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:51335/collection1/
   [junit4]   2> 193715 T700 C:control_collection S:shard1 C54 P51335 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 193715 T700 C:control_collection S:shard1 C54 P51335 oasc.SyncStrategy.syncToMe http://127.0.0.1:51335/collection1/ has no replicas
   [junit4]   2> 193716 T700 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:51335/collection1/ shard1
   [junit4]   2> 193716 T692 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 193716 T700 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 193716 T704 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 193718 T666 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 193718 T666 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 193718 T666 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 193719 T666 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 193720 T707 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@267ceb1 name:ZooKeeperConnection Watcher:127.0.0.1:51328/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 193721 T666 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 193721 T692 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 193722 T666 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 193722 T693 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51335",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 193724 T693 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 193724 T693 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 193725 T704 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 193725 T692 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 193727 T692 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 193728 T693 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 193729 T692 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 193729 T704 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 193771 T700 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:51335/collection1/ and leader is http://127.0.0.1:51335/collection1/
   [junit4]   2> 193772 T700 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:51335
   [junit4]   2> 193772 T700 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 193772 T700 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 193772 T700 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 193773 T692 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 193774 T693 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:51335_",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51335",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 4
   [junit4]   2> 193775 T693 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:51335_",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51335",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection"}
   [junit4]   2> 193878 T692 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 193878 T704 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 195770 T666 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/shard-1-001/cores/collection1
   [junit4]   2> 195774 T666 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/shard-1-001
   [junit4]   2> 195775 T666 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 195780 T666 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:51357
   [junit4]   2> 195780 T666 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solrconfig=solrconfig.xml, hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/shard-1-001/cores, hostPort=51356, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/tempDir-001/jetty1}
   [junit4]   2> 195781 T666 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 195781 T666 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/shard-1-001/'
   [junit4]   2> 195843 T666 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/shard-1-001/solr.xml
   [junit4]   2> 195918 T666 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/shard-1-001/cores
   [junit4]   2> 195918 T666 oasc.CoreContainer.<init> New CoreContainer 769964583
   [junit4]   2> 195919 T666 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/shard-1-001/]
   [junit4]   2> 195919 T666 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/shard-1-001/lib
   [junit4]   2> 195920 T666 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/shard-1-001/lib).
   [junit4]   2> 195935 T666 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 195936 T666 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 195936 T666 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 195937 T666 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 195937 T666 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 195938 T666 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 195938 T666 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 195938 T666 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 195939 T666 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 195939 T666 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 195940 T666 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 195940 T666 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 195941 T666 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 195941 T666 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 195942 T666 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 195942 T666 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51328/solr
   [junit4]   2> 195943 T666 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 195943 T666 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 195944 T666 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 195946 T719 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2a53eb30 name:ZooKeeperConnection Watcher:127.0.0.1:51328 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 195947 T666 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 195949 T666 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 195951 T666 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 195952 T722 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3f0480e1 name:ZooKeeperConnection Watcher:127.0.0.1:51328/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 195952 T666 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 195957 T666 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 196963 T666 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51356_
   [junit4]   2> 196964 T666 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:51356_
   [junit4]   2> 196966 T666 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 196969 T666 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/shard-1-001/cores
   [junit4]   2> 196971 T666 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/shard-1-001/cores/collection1, loadOnStartup=true, name=collection1, collection=collection1, shard=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/shard-1-001/cores/collection1/, schema=schema.xml, transient=false, dataDir=data/, coreNodeName=}
   [junit4]   2> 196971 T666 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/shard-1-001/cores/collection1/
   [junit4]   2> 196971 T666 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 196974 T723 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 196974 T723 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 196975 T692 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 196976 T723 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 196976 T723 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 196976 T693 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:51356_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51356",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 196976 T723 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 196977 T693 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:51356_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51356",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1"}
   [junit4]   2> 196977 T693 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 196977 T693 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 196977 T723 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 196978 T723 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 196978 T693 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 196979 T722 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 196980 T722 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 197978 T723 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 197979 T723 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 197980 T723 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 197980 T723 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 197981 T723 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 197981 T723 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/shard-1-001/cores/collection1/'
   [junit4]   2> 198000 T723 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 198003 T723 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 198017 T723 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 198041 T723 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 198095 T723 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 198196 T723 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 198209 T723 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 198660 T723 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 198678 T723 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 198682 T723 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 198697 T723 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 198702 T723 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 198706 T723 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 198707 T723 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 198707 T723 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 198708 T723 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 198708 T723 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 198709 T723 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 198709 T723 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 198709 T723 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 198710 T723 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 198710 T723 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6e7ce201
   [junit4]   2> 198711 T723 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/shard-1-001/cores/collection1/data
   [junit4]   2> 198712 T723 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 198712 T723 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 198713 T723 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest FC97E93C8C0589AB-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 198713 T723 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=14, maxMergeAtOnceExplicit=25, maxMergedSegmentMB=5.734375, floorSegmentMB=0.28125, forceMergeDeletesPctAllowed=2.459418236654205, segmentsPerTier=42.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7603132523207807
   [junit4]   2> 198714 T723 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@50487964 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4619bd69),segFN=segments_1,generation=1}
   [junit4]   2> 198715 T723 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 198724 T723 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 198725 T723 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 198725 T723 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 198726 T723 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 198726 T723 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 198726 T723 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 198727 T723 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 198728 T723 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 198728 T723 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 198756 T723 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 198759 T723 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 198762 T723 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 198765 T723 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 198776 T723 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update/csv,standard,/replication,/admin/plugins,/update/json,/update,/admin/system,/admin/logging,/admin/ping,/update/json/docs,/admin/properties,/get,/admin/mbeans,/admin/file,/config,/admin/threads,/admin/segments,/admin/luke,/schema
   [junit4]   2> 198778 T723 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 198780 T723 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 198780 T723 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 198782 T723 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 198782 T723 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 198784 T723 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=37, maxMergeAtOnceExplicit=14, maxMergedSegmentMB=6.4130859375, floorSegmentMB=0.2431640625, forceMergeDeletesPctAllowed=21.323455194532404, segmentsPerTier=14.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.42610527605398063
   [junit4]   2> 198785 T723 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@50487964 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4619bd69),segFN=segments_1,generation=1}
   [junit4]   2> 198785 T723 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 198786 T723 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@d2dbf4[collection1] main
   [junit4]   2> 198786 T723 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 198787 T723 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 198788 T723 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 198788 T723 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 198788 T723 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 198789 T723 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 198789 T723 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 198790 T723 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 198790 T723 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 198791 T723 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 198792 T724 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@d2dbf4[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 198792 T723 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 198793 T727 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:51356 collection:collection1 shard:shard1
   [junit4]   2> 198794 T666 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 198795 T666 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 198795 T727 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 198798 T727 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 198800 T692 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 198800 T727 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 198800 T727 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C55 name=collection1 org.apache.solr.core.SolrCore@3bcb5214 url=http://127.0.0.1:51356/collection1 node=127.0.0.1:51356_ C55_STATE=coll:collection1 core:collection1 props:{node_name=127.0.0.1:51356_, state=down, core=collection1, base_url=http://127.0.0.1:51356}
   [junit4]   2> 198801 T727 C:collection1 S:shard1 C55 P51356 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:51356/collection1/
   [junit4]   2> 198801 T693 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 198801 T727 C:collection1 S:shard1 C55 P51356 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 198802 T727 C:collection1 S:shard1 C55 P51356 oasc.SyncStrategy.syncToMe http://127.0.0.1:51356/collection1/ has no replicas
   [junit4]   2> 198802 T727 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:51356/collection1/ shard1
   [junit4]   2> 198802 T693 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 198802 T727 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 198803 T722 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 198804 T722 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 198806 T692 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 198807 T693 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51356",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 198808 T693 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 198808 T722 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 198809 T722 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 198857 T727 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:51356/collection1/ and leader is http://127.0.0.1:51356/collection1/
   [junit4]   2> 198858 T727 C:collection1 S:shard1 oasc.ZkController.re

[...truncated too long message...]

on S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:14200 14200
   [junit4]   2> 1647786 T4415 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:14200 14200
   [junit4]   2> 1647787 T4415 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 		2	/solr/collections/collection1/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1647788 T4414 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:32681/i_bl/sd
   [junit4]   2> 1647788 T4414 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:52593/i_bl/sd
   [junit4]   2> 1647788 T4414 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 6 connections to: http://127.0.0.1:40258/i_bl/sd
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=FC97E93C8C0589AB -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=es_AR -Dtests.timezone=Etc/GMT-4 -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   71.1s J2 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:40258/i_bl/sd/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([FC97E93C8C0589AB:74C3D6E622F9E453]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:40258/i_bl/sd/collection1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 49 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 54 more
   [junit4]   2> 1647801 T4414 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest FC97E93C8C0589AB-001
   [junit4]   2> 71067 T4413 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Lucene50, sim=DefaultSimilarity, locale=es_AR, timezone=Etc/GMT-4
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=108653768,total=382205952
   [junit4]   2> NOTE: All tests run in this JVM: [DirectUpdateHandlerOptimizeTest, TestStandardQParsers, SpellingQueryConverterTest, DocumentAnalysisRequestHandlerTest, UUIDFieldTest, TestSolrConfigHandler, ResourceLoaderTest, HdfsChaosMonkeySafeLeaderTest, TestCollationFieldDocValues, TestUpdate, TestUtils, DistribDocExpirationUpdateProcessorTest, TestStressReorder, SyncSliceTest, SharedFSAutoReplicaFailoverTest, TestSolrJ, TestSchemaVersionResource, TestSchemaResource, ShardSplitTest, TestSolr4Spatial, TestFastLRUCache, TestHashQParserPlugin, EnumFieldTest, TestManagedSchemaDynamicFieldResource, CSVRequestHandlerTest, SaslZkACLProviderTest, DistributedTermsComponentTest, CloudMLTQParserTest, CoreAdminRequestStatusTest, TestSerializedLuceneMatchVersion, ParsingFieldUpdateProcessorsTest, HardAutoCommitTest, TestBlendedInfixSuggestions, TestManagedSchemaFieldTypeResource, TestClusterStateMutator, CoreAdminHandlerTest, TestFastWriter, TestDistributedSearch, TestFuzzyAnalyzedSuggestions, DistributedQueryComponentCustomSortTest, TestDistributedMissingSort, TestTolerantSearch, SpatialHeatmapFacetsTest, TestFoldingMultitermQuery, TestReloadAndDeleteDocs, CursorMarkTest, TestAtomicUpdateErrorCases, TestGroupingSearch, TestRemoteStreaming, TestReplicaProperties, TestStressUserVersions, TestDefaultSimilarityFactory, TestSolrConfigHandlerCloud, TestMaxScoreQueryParser, ExternalCollectionsTest, CoreMergeIndexesAdminHandlerTest, DocExpirationUpdateProcessorFactoryTest, HdfsUnloadDistributedZkTest, TestCloudSchemaless, CustomCollectionTest, TestSolrConfigHandlerConcurrent, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J2 in 71.36s, 1 test, 1 error <<< FAILURES!

[...truncated 690 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 62 ignored (22 assumptions)

Total time: 53 minutes 21 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 202810468 bytes
Compression is 0.0%
Took 38 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2859 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2859/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:39463/l_w/m/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:39463/l_w/m/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([1C620EC4DA0E131B:9436311E74F27EE3]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([1C620EC4DA0E131B:9436311E74F27EE3]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([1C620EC4DA0E131B:9436311E74F27EE3]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9738 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/init-core-data-001
   [junit4]   2> 779457 T1715 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 779462 T1715 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 779462 T1716 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 779562 T1715 oasc.ZkTestServer.run start zk server on port:38739
   [junit4]   2> 779563 T1715 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 779563 T1715 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 779568 T1723 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5b073ece name:ZooKeeperConnection Watcher:127.0.0.1:38739 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 779568 T1715 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 779568 T1715 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 779568 T1715 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 779571 T1715 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 779572 T1715 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 779573 T1726 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4b5a4214 name:ZooKeeperConnection Watcher:127.0.0.1:38739/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 779573 T1715 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 779574 T1715 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 779574 T1715 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 779576 T1715 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 779577 T1715 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 779578 T1715 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 779579 T1715 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 779580 T1715 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 779582 T1715 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 779583 T1715 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 779686 T1715 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 779686 T1715 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 779688 T1715 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 779688 T1715 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 779690 T1715 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 779690 T1715 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 779691 T1715 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 779692 T1715 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 779693 T1715 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 779694 T1715 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 779695 T1715 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 779696 T1715 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 779697 T1715 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 779698 T1715 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 779699 T1715 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 779699 T1715 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 779701 T1715 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 779701 T1715 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 781166 T1715 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/control-001/cores/collection1
   [junit4]   2> 781169 T1715 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 781173 T1715 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:38746
   [junit4]   2> 781173 T1715 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/control-001/cores, hostContext=/, hostPort=38745}
   [junit4]   2> 781174 T1715 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 781174 T1715 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/control-001/'
   [junit4]   2> 781203 T1715 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/control-001/solr.xml
   [junit4]   2> 781238 T1715 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/control-001/cores
   [junit4]   2> 781238 T1715 oasc.CoreContainer.<init> New CoreContainer 330443851
   [junit4]   2> 781238 T1715 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/control-001/]
   [junit4]   2> 781239 T1715 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/control-001/lib
   [junit4]   2> 781239 T1715 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/control-001/lib).
   [junit4]   2> 781248 T1715 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 781249 T1715 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 781249 T1715 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 781249 T1715 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 781249 T1715 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 781250 T1715 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 781250 T1715 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 781250 T1715 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 781250 T1715 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 781250 T1715 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 781251 T1715 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 781251 T1715 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 781251 T1715 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 781252 T1715 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 781252 T1715 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 781252 T1715 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:38739/solr
   [junit4]   2> 781252 T1715 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 781253 T1715 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 781253 T1715 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 781256 T1738 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@40783dc0 name:ZooKeeperConnection Watcher:127.0.0.1:38739 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 781256 T1715 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 781257 T1715 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 781259 T1715 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 781260 T1741 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@35df312f name:ZooKeeperConnection Watcher:127.0.0.1:38739/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 781260 T1715 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 781262 T1715 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 781264 T1715 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 781265 T1715 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 781267 T1715 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 781269 T1715 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 781271 T1715 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 781272 T1715 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 781273 T1715 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 781274 T1715 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38745_
   [junit4]   2> 781275 T1715 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:38745_
   [junit4]   2> 781276 T1715 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 781277 T1715 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 781278 T1715 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 781280 T1715 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:38745_
   [junit4]   2> 781280 T1715 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 781281 T1715 oasc.Overseer.start Overseer (id=93566953385558019-127.0.0.1:38745_-n_0000000000) starting
   [junit4]   2> 781283 T1715 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 781288 T1715 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 781288 T1743 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 781288 T1715 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 781289 T1742 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 781292 T1715 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/control-001/cores
   [junit4]   2> 781293 T1715 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {transient=false, collection=control_collection, name=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/control-001/cores/collection1, dataDir=data/, shard=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/control-001/cores/collection1/, coreNodeName=, config=solrconfig.xml, schema=schema.xml, loadOnStartup=true}
   [junit4]   2> 781293 T1715 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/control-001/cores/collection1/
   [junit4]   2> 781294 T1715 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 781294 T1745 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 781295 T1745 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 781295 T1741 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 781295 T1745 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 781296 T1742 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:38745_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38745",
   [junit4]   2> 	  "operation":"state"} current state version: 0
   [junit4]   2> 781296 T1742 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:38745_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38745",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 781297 T1742 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 781297 T1742 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 781298 T1741 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 782296 T1745 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 782297 T1745 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 782298 T1745 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 782298 T1745 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 782299 T1745 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 782299 T1745 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/control-001/cores/collection1/'
   [junit4]   2> 782316 T1745 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 782319 T1745 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 782330 T1745 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 782357 T1745 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 782420 T1745 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 782522 T1745 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 782536 T1745 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 782928 T1745 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 782939 T1745 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 782941 T1745 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 782951 T1745 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 782954 T1745 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 782957 T1745 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 782958 T1745 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 782958 T1745 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 782958 T1745 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 782959 T1745 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 782959 T1745 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 782959 T1745 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 782960 T1745 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 782960 T1745 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/control-001/cores/collection1/data/
   [junit4]   2> 782960 T1745 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@237b8408
   [junit4]   2> 782961 T1745 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/control-001/cores/collection1/data
   [junit4]   2> 782961 T1745 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/control-001/cores/collection1/data/index/
   [junit4]   2> 782961 T1745 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 782961 T1745 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/control-001/cores/collection1/data/index
   [junit4]   2> 782962 T1745 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=492055204, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 782962 T1745 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@69464677 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4bc64a0),segFN=segments_1,generation=1}
   [junit4]   2> 782963 T1745 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 782967 T1745 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 782967 T1745 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 782967 T1745 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 782967 T1745 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 782968 T1745 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 782968 T1745 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 782968 T1745 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 782968 T1745 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 782969 T1745 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 782983 T1745 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 782985 T1745 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 782987 T1745 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 782988 T1745 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 782993 T1745 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update/csv,/admin/threads,/admin/file,/admin/logging,/admin/ping,/admin/system,/admin/plugins,/update,/admin/luke,/update/json/docs,/admin/mbeans,/admin/segments,/admin/properties,/get,/config,standard,/replication,/schema,/update/json
   [junit4]   2> 782995 T1745 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 782996 T1745 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 782996 T1745 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 782997 T1745 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 782998 T1745 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 782998 T1745 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=22, maxMergeAtOnceExplicit=33, maxMergedSegmentMB=11.60546875, floorSegmentMB=0.564453125, forceMergeDeletesPctAllowed=21.28685788194332, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8585640263505205
   [junit4]   2> 782999 T1745 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@69464677 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4bc64a0),segFN=segments_1,generation=1}
   [junit4]   2> 782999 T1745 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 782999 T1745 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@6a3278b8[collection1] main
   [junit4]   2> 782999 T1745 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 783000 T1745 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 783000 T1745 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 783001 T1745 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 783001 T1745 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 783001 T1745 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 783002 T1745 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 783002 T1745 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 783002 T1745 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 783002 T1745 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 783003 T1746 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6a3278b8[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 783004 T1745 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 783005 T1749 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:38745 collection:control_collection shard:shard1
   [junit4]   2> 783005 T1715 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 783005 T1715 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 783006 T1749 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 783009 T1715 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 783010 T1715 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 783011 T1749 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 783011 T1753 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@27e10af5 name:ZooKeeperConnection Watcher:127.0.0.1:38739/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 783011 T1715 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 783012 T1715 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 783012 T1715 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 783012 T1741 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 783014 T1749 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 783014 T1749 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C461 name=collection1 org.apache.solr.core.SolrCore@78a2a274 url=http://127.0.0.1:38745/collection1 node=127.0.0.1:38745_ C461_STATE=coll:control_collection core:collection1 props:{state=down, node_name=127.0.0.1:38745_, core=collection1, base_url=http://127.0.0.1:38745}
   [junit4]   2> 783014 T1749 C:control_collection S:shard1 C461 P38745 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:38745/collection1/
   [junit4]   2> 783014 T1742 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 783014 T1749 C:control_collection S:shard1 C461 P38745 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 783015 T1749 C:control_collection S:shard1 C461 P38745 oasc.SyncStrategy.syncToMe http://127.0.0.1:38745/collection1/ has no replicas
   [junit4]   2> 783015 T1749 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:38745/collection1/ shard1
   [junit4]   2> 783015 T1749 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 783015 T1753 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 783016 T1741 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 783017 T1715 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 783020 T1741 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 783021 T1742 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38745",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 783124 T1753 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 783124 T1741 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 783173 T1749 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:38745/collection1/ and leader is http://127.0.0.1:38745/collection1/
   [junit4]   2> 783174 T1749 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:38745
   [junit4]   2> 783174 T1749 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 783174 T1749 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 783174 T1749 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 783176 T1741 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 783177 T1742 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:38745_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38745",
   [junit4]   2> 	  "operation":"state"} current state version: 3
   [junit4]   2> 783177 T1742 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:38745_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38745",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 783281 T1741 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 783281 T1753 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 784734 T1715 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/shard-1-001/cores/collection1
   [junit4]   2> 784737 T1715 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/shard-1-001
   [junit4]   2> 784738 T1715 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 784740 T1715 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:38753
   [junit4]   2> 784741 T1715 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, solrconfig=solrconfig.xml, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/tempDir-001/jetty1, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/shard-1-001/cores, hostPort=38752}
   [junit4]   2> 784741 T1715 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 784741 T1715 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/shard-1-001/'
   [junit4]   2> 784791 T1715 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/shard-1-001/solr.xml
   [junit4]   2> 784833 T1715 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/shard-1-001/cores
   [junit4]   2> 784834 T1715 oasc.CoreContainer.<init> New CoreContainer 1078472374
   [junit4]   2> 784834 T1715 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/shard-1-001/]
   [junit4]   2> 784834 T1715 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/shard-1-001/lib
   [junit4]   2> 784835 T1715 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/shard-1-001/lib).
   [junit4]   2> 784844 T1715 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 784845 T1715 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 784845 T1715 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 784845 T1715 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 784845 T1715 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 784846 T1715 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 784846 T1715 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 784846 T1715 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 784846 T1715 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 784847 T1715 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 784847 T1715 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 784847 T1715 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 784848 T1715 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 784848 T1715 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 784848 T1715 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 784848 T1715 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:38739/solr
   [junit4]   2> 784849 T1715 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 784849 T1715 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 784849 T1715 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 784851 T1765 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1fed8437 name:ZooKeeperConnection Watcher:127.0.0.1:38739 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 784852 T1715 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 784853 T1715 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 784855 T1715 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 784856 T1768 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6b0cc67a name:ZooKeeperConnection Watcher:127.0.0.1:38739/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 784856 T1715 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 784860 T1715 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 785864 T1715 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38752_
   [junit4]   2> 785865 T1715 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:38752_
   [junit4]   2> 785869 T1715 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 785871 T1715 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/shard-1-001/cores
   [junit4]   2> 785873 T1715 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {dataDir=data/, loadOnStartup=true, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/shard-1-001/cores/collection1/, config=solrconfig.xml, shard=, coreNodeName=, name=collection1, schema=schema.xml, transient=false, collection=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/shard-1-001/cores/collection1}
   [junit4]   2> 785874 T1715 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/shard-1-001/cores/collection1/
   [junit4]   2> 785874 T1715 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 785876 T1769 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 785876 T1769 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 785876 T1741 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 785876 T1769 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 785877 T1742 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:38752_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38752",
   [junit4]   2> 	  "operation":"state"} current state version: 4
   [junit4]   2> 785878 T1742 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:38752_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38752",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 785878 T1742 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 785878 T1742 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 785880 T1768 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 785880 T1753 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 785880 T1741 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 786877 T1769 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 786878 T1769 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 786878 T1769 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 786879 T1769 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 786879 T1769 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 786879 T1769 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/shard-1-001/cores/collection1/'
   [junit4]   2> 786892 T1769 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 786894 T1769 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 786902 T1769 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 786918 T1769 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 786955 T1769 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 787057 T1769 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 787066 T1769 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 787382 T1769 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 787396 T1769 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 787398 T1769 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 787407 T1769 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 787410 T1769 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 787414 T1769 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 787414 T1769 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 787414 T1769 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 787415 T1769 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 787415 T1769 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 787415 T1769 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 787416 T1769 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 787416 T1769 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 787416 T1769 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 787416 T1769 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@237b8408
   [junit4]   2> 787417 T1769 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/shard-1-001/cores/collection1/data
   [junit4]   2> 787417 T1769 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 787417 T1769 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 787418 T1769 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E131B-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 787419 T1769 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=492055204, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 787420 T1769 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@448aec0d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7c8f6ca2),segFN=segments_1,generation=1}
   [junit4]   2> 787420 T1769 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 787426 T1769 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 787426 T1769 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 787426 T1769 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 787427 T1769 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 787427 T1769 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 787427 T1769 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 787427 T1769 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 787428 T1769 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 787428 T1769 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 787447 T1769 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 787450 T1769 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 787452 T1769 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 787454 T1769 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 787461 T1769 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update/csv,/admin/threads,/admin/file,/admin/logging,/admin/ping,/admin/system,/admin/plugins,/update,/admin/luke,/update/json/docs,/admin/mbeans,/admin/segments,/admin/properties,/get,/config,standard,/replication,/schema,/update/json
   [junit4]   2> 787463 T1769 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 787464 T1769 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 787465 T1769 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 787466 T1769 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 787466 T1769 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 787467 T1769 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=22, maxMergeAtOnceExplicit=33, maxMergedSegmentMB=11.60546875, floorSegmentMB=0.564453125, forceMergeDeletesPctAllowed=21.28685788194332, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8585640263505205
   [junit4]   2> 787468 T1769 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@448aec0d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7c8f6ca2),segFN=segments_1,generation=1}
   [junit4]   2> 787468 T1769 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 787468 T1769 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@68ba9b29[collection1] main
   [junit4]   2> 787469 T1769 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 787470 T1769 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 787470 T1769 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 787470 T1769 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 787470 T1769 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 787470 T1769 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 787471 T1769 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 787471 T1769 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 787471 T1769 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 787471 T1769 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 787473 T1770 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@68ba9b29[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 787473 T1769 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 787474 T1773 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:38752 collection:collection1 shard:shard1
   [junit4]   2> 787474 T1715 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 787474 T1715 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 787474 T1773 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 787478 T1773 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 787479 T1741 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 787479 T1773 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 787480 T1773 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 787480 T1742 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> ASYNC  NEW_CORE C462 name=collection1 org.apache.solr.core.SolrCore@5a23e8d5 url=http://127.0.0.1:38752/collection1 node=127.0.0.1:38752_ C462_STATE=coll:collection1 core:collection1 props:{state=down, node_name=127.0.0.1:38752_, core=collection1, base_url=http://127.0.0.1:38752}
   [junit4]   2> 787480 T1773 C:collection1 S:shard1 C462 P38752 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:38752/collection1/
   [junit4]   2> 787480 T1773 C:collection1 S:shard1 C462 P38752 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 787480 T1773 C:collection1 S:shard1 C462 P38752 oasc.SyncStrategy.syncToMe http://127.0.0.1:38752/collection1/ has no replicas
   [junit4]   2> 787481 T1773 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:38752/collection1/ shard1
   [junit4]   2> 787481 T1773 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 787481 T1741 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 787481 T1768 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 787481 T1753 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 787484 T1741 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 787485 T1742 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38752",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 787587 T1741 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 787587 T1768 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 787587 T1753 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 787637 T1773 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:38752/collection1/ and leader is http://127.0.0.1:38752/collection1/
   [junit4]   2> 787638 T1773 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:38752
   [junit4]   2> 787638 T1773 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 787638 T1773 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 787638 T1773 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 787639 T1741 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 787640 T1742 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:38752_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38752",
   [junit4]   2> 	  "operation":"state"} current state version: 7
   [junit4]   2> 787641 T1742 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:38752_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38752",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 787744 T1741 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 787744 T1768 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 787744 T1753 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 788892 T1715 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1C620EC4DA0E1

[...truncated too long message...]

> 1548234 T5819 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 9 connections to: http://127.0.0.1:56928/_r
   [junit4]   2> 1548234 T5819 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 6 connections to: http://127.0.0.1:57008/_r
   [junit4]   2> 1548234 T5819 C:control_collection S:shard1 c:collection1 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 1548235 T5819 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:56960/_r
   [junit4]   2> 1548235 T5819 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:56953/_r
   [junit4]   2> 1548235 T5819 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:56928/_r
   [junit4]   2> 1548235 T5819 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:57008/_r
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=1C620EC4DA0E131B -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=it -Dtests.timezone=CTT -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR    126s J3 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([1C620EC4DA0E131B:9436311E74F27EE3]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 1548253 T5819 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 1C620EC4DA0E131B-001
   [junit4]   2> 125745 T5818 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {a_t=PostingsFormat(name=Memory doPackFST= true), _version_=PostingsFormat(name=MockRandom), text=PostingsFormat(name=Memory doPackFST= false), intDefault=PostingsFormat(name=MockRandom), id=PostingsFormat(name=LuceneFixedGap), range_facet_l=PostingsFormat(name=Memory doPackFST= true), multiDefault=PostingsFormat(name=Memory doPackFST= true), timestamp=PostingsFormat(name=Memory doPackFST= true)}, docValues:{timestamp=DocValuesFormat(name=Lucene50)}, sim=RandomSimilarityProvider(queryNorm=false,coord=yes): {}, locale=it, timezone=CTT
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=148449568,total=379584512
   [junit4]   2> NOTE: All tests run in this JVM: [HighlighterMaxOffsetTest, StatelessScriptUpdateProcessorFactoryTest, TestJmxIntegration, JSONWriterTest, WordBreakSolrSpellCheckerTest, HdfsLockFactoryTest, TestPivotHelperCode, CSVRequestHandlerTest, SegmentsInfoRequestHandlerTest, CoreAdminHandlerTest, TestAddFieldRealTimeGet, BlockCacheTest, TestSimpleQParserPlugin, SolrCoreTest, TestReqParamsAPI, HdfsBasicDistributedZk2Test, TestDistributedMissingSort, QueryResultKeyTest, RollingRestartTest, TestHashQParserPlugin, DirectUpdateHandlerOptimizeTest, TestBinaryField, BlockDirectoryTest, TestNonDefinedSimilarityFactory, TestManagedSchema, DistributedFacetPivotWhiteBoxTest, TestReRankQParserPlugin, ActionThrottleTest, PrimitiveFieldTypeTest, PreAnalyzedUpdateProcessorTest, FileBasedSpellCheckerTest, UnloadDistributedZkTest, TestRestManager, OutOfBoxZkACLAndCredentialsProvidersTest, OverseerTest, BasicZkTest, DocExpirationUpdateProcessorFactoryTest, TestIBSimilarityFactory, TestAnalyzeInfixSuggestions, HdfsUnloadDistributedZkTest, TestQueryUtils, TestShardHandlerFactory, OpenExchangeRatesOrgProviderTest, SignatureUpdateProcessorFactoryTest, BasicDistributedZk2Test, TestOrdValues, MergeStrategyTest, TestCollationField, TestCloudManagedSchema, CurrencyFieldXmlFileTest, TestBulkSchemaConcurrent, TestPartialUpdateDeduplication, TestSolrConfigHandler, TermsComponentTest, CollectionsAPIDistributedZkTest, TestLMDirichletSimilarityFactory, ExternalCollectionsTest, CoreAdminRequestStatusTest, AsyncMigrateRouteKeyTest, UpdateRequestProcessorFactoryTest, TestCloudManagedSchemaConcurrent, BasicDistributedZkTest, ReplicationFactorTest]
   [junit4] Completed on J3 in 126.05s, 1 test, 1 error <<< FAILURES!

[...truncated 715 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 62 ignored (22 assumptions)

Total time: 53 minutes 28 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 210814757 bytes
Compression is 0.0%
Took 40 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2858 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2858/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:35653/pddwr/oq/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:35653/pddwr/oq/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([54CB20103163D96E:DC9F1FCA9F9FB496]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:11042/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:11042/collection1
	at __randomizedtesting.SeedInfo.seed([54CB20103163D96E:DC9F1FCA9F9FB496]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:11042/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:36306/x_/repfacttest_c8n_1x3_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:36306/x_/repfacttest_c8n_1x3_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([54CB20103163D96E:DC9F1FCA9F9FB496]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:36306/x_/repfacttest_c8n_1x3_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9268 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/init-core-data-001
   [junit4]   2> 46259 T86 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /x_/
   [junit4]   2> 46317 T86 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 46329 T87 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 46731 T86 oasc.ZkTestServer.run start zk server on port:53272
   [junit4]   2> 46944 T86 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 47286 T86 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 47510 T94 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@e75825e name:ZooKeeperConnection Watcher:127.0.0.1:53272 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 47511 T86 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 47512 T86 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 47528 T86 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 47692 T86 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 47693 T86 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 47695 T97 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1a851dc name:ZooKeeperConnection Watcher:127.0.0.1:53272/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 47695 T86 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 47696 T86 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 47706 T86 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 47711 T86 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 47716 T86 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 47719 T86 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 47724 T86 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 47725 T86 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 47732 T86 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 47732 T86 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 47737 T86 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 47738 T86 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 47743 T86 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 47743 T86 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 47749 T86 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 47750 T86 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 47756 T86 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 47756 T86 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 47763 T86 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 47763 T86 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 47768 T86 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 47769 T86 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 47774 T86 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 47774 T86 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 47779 T86 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 47780 T86 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 47785 T86 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 47785 T86 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 49389 T86 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/control-001/cores/collection1
   [junit4]   2> 49393 T86 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 49397 T86 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:12810
   [junit4]   2> 49397 T86 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/tempDir-001/control/data, hostContext=/x_, hostPort=62358, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/control-001/cores}
   [junit4]   2> 49398 T86 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 49398 T86 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/control-001/'
   [junit4]   2> 49431 T86 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/control-001/solr.xml
   [junit4]   2> 49480 T86 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/control-001/cores
   [junit4]   2> 49481 T86 oasc.CoreContainer.<init> New CoreContainer 847657480
   [junit4]   2> 49481 T86 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/control-001/]
   [junit4]   2> 49481 T86 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/control-001/lib
   [junit4]   2> 49482 T86 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/control-001/lib).
   [junit4]   2> 49492 T86 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 49492 T86 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 49493 T86 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 49493 T86 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 49493 T86 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 49493 T86 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 49494 T86 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 49494 T86 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 49494 T86 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 49494 T86 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 49495 T86 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 49495 T86 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 49496 T86 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 49496 T86 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 49496 T86 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 49497 T86 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:53272/solr
   [junit4]   2> 49540 T86 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 49540 T86 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 49541 T86 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 49543 T109 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@36082d97 name:ZooKeeperConnection Watcher:127.0.0.1:53272 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 49543 T86 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 49544 T86 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 49564 T86 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 49566 T112 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@132c9694 name:ZooKeeperConnection Watcher:127.0.0.1:53272/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 49566 T86 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 49584 T86 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 49604 T86 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 49612 T86 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 49616 T86 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 49620 T86 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 49641 T86 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 49644 T86 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 49647 T86 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 49649 T86 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:62358_x_
   [junit4]   2> 49656 T86 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:62358_x_
   [junit4]   2> 49659 T86 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 49662 T86 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 49665 T86 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 49686 T86 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:62358_x_
   [junit4]   2> 49687 T86 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 49689 T86 oasc.Overseer.start Overseer (id=93566018880602115-127.0.0.1:62358_x_-n_0000000000) starting
   [junit4]   2> 49716 T86 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 49769 T86 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 49793 T114 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 49793 T86 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 49818 T113 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 49832 T86 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/control-001/cores
   [junit4]   2> 49848 T86 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {coreNodeName=, schema=schema.xml, name=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/control-001/cores/collection1/, config=solrconfig.xml, collection=control_collection, dataDir=data/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/control-001/cores/collection1, transient=false, shard=, loadOnStartup=true}
   [junit4]   2> 49849 T86 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/control-001/cores/collection1/
   [junit4]   2> 49849 T86 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 49851 T116 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 49852 T116 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 49869 T112 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 49869 T116 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 49876 T113 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62358/x_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:62358_x_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1"} current state version: 0
   [junit4]   2> 49946 T113 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62358/x_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:62358_x_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 49957 T113 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 50017 T113 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 50037 T112 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 50871 T116 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 50871 T116 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 50872 T116 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 50873 T116 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 50874 T116 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 50875 T116 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/control-001/cores/collection1/'
   [junit4]   2> 50931 T116 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 50964 T116 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 51002 T116 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 51026 T116 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 51087 T116 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 51089 T116 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 51127 T116 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 51561 T116 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 51592 T116 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 51597 T116 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 51635 T116 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 51643 T116 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 51650 T116 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 51651 T116 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 51652 T116 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 51652 T116 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 51654 T116 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 51654 T116 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 51655 T116 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 51655 T116 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 51656 T116 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/control-001/cores/collection1/data/
   [junit4]   2> 51657 T116 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7d800ca2
   [junit4]   2> 51680 T116 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/control-001/cores/collection1/data
   [junit4]   2> 51680 T116 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/control-001/cores/collection1/data/index/
   [junit4]   2> 51680 T116 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 51681 T116 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/control-001/cores/collection1/data/index
   [junit4]   2> 51682 T116 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=12, maxMergeAtOnceExplicit=18, maxMergedSegmentMB=1.513671875, floorSegmentMB=0.5322265625, forceMergeDeletesPctAllowed=0.24743097973329076, segmentsPerTier=40.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.14276667757285663
   [junit4]   2> 51683 T116 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@e980302 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1ded7fca),segFN=segments_1,generation=1}
   [junit4]   2> 51684 T116 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 51696 T116 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 51801 T116 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 51801 T116 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 51802 T116 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 51802 T116 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 51802 T116 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 51803 T116 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 51804 T116 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 51805 T116 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 51837 T116 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 51852 T116 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 51856 T116 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 51859 T116 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 51875 T116 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: standard,/admin/logging,/replication,/schema,/admin/properties,/config,/update/json/docs,/admin/segments,/get,/update/csv,/admin/threads,/admin/file,/admin/luke,/update,/update/json,/admin/ping,/admin/system,/admin/plugins,/admin/mbeans
   [junit4]   2> 51878 T116 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 51881 T116 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 51881 T116 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 51884 T116 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 51884 T116 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 51886 T116 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=39, maxMergedSegmentMB=82.7783203125, floorSegmentMB=2.142578125, forceMergeDeletesPctAllowed=28.495659768757626, segmentsPerTier=23.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.43523712462718445
   [junit4]   2> 51887 T116 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@e980302 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1ded7fca),segFN=segments_1,generation=1}
   [junit4]   2> 51887 T116 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 51888 T116 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@77763102[collection1] main
   [junit4]   2> 51888 T116 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 51890 T116 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 51891 T116 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 51911 T116 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 51911 T116 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 51911 T116 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 51912 T116 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 51913 T116 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 51913 T116 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 51914 T116 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 51917 T117 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@77763102[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 51923 T116 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 51924 T120 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:62358/x_ collection:control_collection shard:shard1
   [junit4]   2> 51925 T86 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 51925 T86 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 51981 T120 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 51993 T120 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 51997 T112 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 51999 T120 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 51999 T120 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 52000 T113 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 52014 T86 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 52015 T86 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> ASYNC  NEW_CORE C13 name=collection1 org.apache.solr.core.SolrCore@2a3c0a0d url=http://127.0.0.1:62358/x_/collection1 node=127.0.0.1:62358_x_ C13_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:62358/x_, node_name=127.0.0.1:62358_x_, state=down, core=collection1}
   [junit4]   2> 52015 T120 C:control_collection S:shard1 C13 P62358 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:62358/x_/collection1/
   [junit4]   2> 52016 T120 C:control_collection S:shard1 C13 P62358 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 52017 T120 C:control_collection S:shard1 C13 P62358 oasc.SyncStrategy.syncToMe http://127.0.0.1:62358/x_/collection1/ has no replicas
   [junit4]   2> 52017 T120 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:62358/x_/collection1/ shard1
   [junit4]   2> 52019 T124 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@72afb185 name:ZooKeeperConnection Watcher:127.0.0.1:53272/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 52019 T86 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 52020 T86 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 52020 T86 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 52026 T112 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 52029 T124 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 52032 T120 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 52041 T86 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 52042 T86 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 52042 T86 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 52043 T86 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 52046 T112 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 52046 T127 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4e569945 name:ZooKeeperConnection Watcher:127.0.0.1:53272/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 52047 T86 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 52048 T86 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 52048 T113 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62358/x_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 52056 T113 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 52057 T113 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 52059 T124 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 52059 T112 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 52073 T112 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 52075 T113 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 52078 T124 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 52078 T112 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 52099 T120 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:62358/x_/collection1/ and leader is http://127.0.0.1:62358/x_/collection1/
   [junit4]   2> 52099 T120 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:62358/x_
   [junit4]   2> 52100 T120 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 52100 T120 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 52101 T120 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 52118 T112 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 52125 T113 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62358/x_",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:62358_x_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core":"collection1"} current state version: 4
   [junit4]   2> 52127 T113 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62358/x_",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:62358_x_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 52245 T112 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 52246 T124 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 55513 T86 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/shard-1-001/cores/collection1
   [junit4]   2> 55516 T86 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/shard-1-001
   [junit4]   2> 55517 T86 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 55520 T86 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:36298
   [junit4]   2> 55520 T86 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solrconfig=solrconfig.xml, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/tempDir-001/jetty1, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/shard-1-001/cores, hostContext=/x_, hostPort=36297}
   [junit4]   2> 55521 T86 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 55522 T86 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/shard-1-001/'
   [junit4]   2> 55566 T86 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/shard-1-001/solr.xml
   [junit4]   2> 55626 T86 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/shard-1-001/cores
   [junit4]   2> 55626 T86 oasc.CoreContainer.<init> New CoreContainer 1555002403
   [junit4]   2> 55627 T86 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/shard-1-001/]
   [junit4]   2> 55627 T86 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/shard-1-001/lib
   [junit4]   2> 55628 T86 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/shard-1-001/lib).
   [junit4]   2> 55641 T86 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 55641 T86 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 55642 T86 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 55642 T86 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 55643 T86 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 55643 T86 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 55643 T86 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 55644 T86 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 55644 T86 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 55644 T86 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 55645 T86 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 55645 T86 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 55646 T86 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 55646 T86 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 55647 T86 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 55647 T86 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:53272/solr
   [junit4]   2> 55647 T86 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 55648 T86 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 55649 T86 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 55652 T139 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7167be2f name:ZooKeeperConnection Watcher:127.0.0.1:53272 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 55652 T86 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 55653 T86 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 55656 T86 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 55658 T142 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@48ec5a5d name:ZooKeeperConnection Watcher:127.0.0.1:53272/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 55659 T86 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 55668 T86 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 56679 T86 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36297_x_
   [junit4]   2> 56680 T86 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:36297_x_
   [junit4]   2> 56684 T86 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 56692 T86 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/shard-1-001/cores
   [junit4]   2> 56694 T86 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {schema=schema.xml, config=solrconfig.xml, name=collection1, shard=, collection=collection1, loadOnStartup=true, transient=false, dataDir=data/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/shard-1-001/cores/collection1, coreNodeName=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/shard-1-001/cores/collection1/}
   [junit4]   2> 56694 T86 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/shard-1-001/cores/collection1/
   [junit4]   2> 56694 T86 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 56698 T143 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 56698 T143 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 56701 T112 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 56702 T143 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 56702 T143 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 56702 T113 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:36297/x_",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:36297_x_",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1"} current state version: 5
   [junit4]   2> 56703 T143 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 56703 T113 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:36297/x_",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:36297_x_",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 56704 T113 oasco.ReplicaMutator.updateState Collection already exists with numShards=3
   [junit4]   2> 56704 T113 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 56707 T113 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 56713 T143 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 56713 T143 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 56713 T143 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 56714 T143 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 56714 T143 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 56715 T143 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 56716 T143 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 56716 T143 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/shard-1-001/cores/collection1/'
   [junit4]   2> 56729 T143 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 56732 T143 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 56742 T143 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 56760 T143 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 56799 T143 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 56801 T143 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 56814 T143 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 57177 T143 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 57192 T143 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 57194 T143 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 57228 T143 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 57235 T143 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 57240 T143 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 57242 T143 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 57242 T143 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 57243 T143 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 57245 T143 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 57245 T143 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 57245 T143 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 57246 T143 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 57246 T143 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 57247 T143 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7d800ca2
   [junit4]   2> 57248 T143 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/shard-1-001/cores/collection1/data
   [junit4]   2> 57248 T143 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 57249 T143 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 57249 T143 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 54CB20103163D96E-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 57250 T143 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=12, maxMergeAtOnceExplicit=18, maxMergedSegmentMB=1.513671875, floorSegmentMB=0.5322265625, forceMergeDeletesPctAllowed=0.24743097973329076, segmentsPerTier=40.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.14276667757285663
   [junit4]   2> 57251 T143 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@265f9b34 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4ec6869a),segFN=segments_1,generation=1}
   [junit4]   2> 57252 T143 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 57268 T143 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 57269 T143 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 57269 T143 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 57270 T143 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 57270 T143 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 57270 T143 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 57271 T143 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 57272 T143 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 57272 T143 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 57294 T143 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 57299 T143 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 57303 T143 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 57307 T143 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 57346 T143 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: standard,/admin/logging,/replication,/schema,/admin/properties,/config,/update/json/docs,/admin/segments,/get,/update/csv,/admin/threads,/admin/file,/admin/luke,/update,/update/json,/admin/ping,/admin/system,/admin/plugins,/admin/mbeans
   [junit4]   2> 57349 T143 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 57351 T143 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 57351 T143 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 57353 T143 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 57353 T143 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 57355 T143 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=39, maxMergedSegmentMB=82.7783203125, floorSegmentMB=2.142578125, forceMergeDeletesPctAllowed=28.495659768757626, segmentsPerTier=23.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.43523712462718445
   [junit4]   2> 57356 T143 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@265f9b34 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4ec6869a),segFN=segments_1,generation=1}
   [junit4]   2> 57357 T143 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 57357 T143 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@be1e46d[collection1] main
   [junit4]   2> 57358 T143 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 57360 T143 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 57360 T143 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 57361 T143 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 57361 T143 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 57362 T143 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 57363 T143 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 57363 T143 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 57364 T143 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 57364 T143 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 57368 T144 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@be1e46d[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 57369 T143 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 57370 T147 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:36297/x_ collection:collection1 shard:shard1
   [junit4]   2> 57371 T86 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 57371 T86 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 57372 T147 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 57383 T147 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 57387 T112 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 57388 T147 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 57388 T147 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C14 name=collection1 org.apache.solr.core.SolrCore@378f8dfa url=http://127.0.0.1:36297/x_/collection1 node=127.0.0.1:36297_x_ C14_STATE=coll:collection1 core:collection1 props:{base_url=http://127.0.0.1:36297/x_, node_name=127.0.0.1:36297_x_, state=down, core=collection1}
   [junit4]   2> 57388 T147 C:collection1 S:shard1 C14 P36297 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:36297/x_/collection1/
   [junit4]   2> 57389 T113 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 57389 T147 C:collection1 S:shard1 C14 P36297 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 57390 T147 C:collection1 S:shard1 C14 P36297 oasc.SyncStrategy.syncToMe http://127.0.0.1:36297/x_/collection1/ has no replicas
   [junit4]   2> 57391 T147 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:36297/x_/collection1/ shard1
   [junit4]   2> 57391 T147 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 57393 T113 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 57394 T142 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 57396 T142 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 57402 T113 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:36297/x_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 57404 T113 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 57405 T142 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 57407 T142 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 57411 T112 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 57452 T147 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:36297/x_/collection1/ and leader is http://127.0.0.1:36297/x_/collection1/
   [junit4]   2> 57453 T147 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:36297/x_
   [junit4]   2> 57453 T147 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 57454 T147 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 57454 T147 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 57456 T112 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 57458 T113 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:36297/x_",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:36297_x_",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core":"collection1"} current state version: 5
   [junit4]   2> 57460 T113 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "base_url":"http://

[...truncated too long message...]

  [junit4]   2> 		at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:291)
   [junit4]   2> 		at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:486)
   [junit4]   2> 		at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:455)
   [junit4]   2> 		at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:442)
   [junit4]   2> 		at org.apache.solr.cloud.ShardLeaderElectionContextBase$1.execute(ElectionContext.java:146)
   [junit4]   2> 		at org.apache.solr.common.util.RetryUtil.retryOnThrowable(RetryUtil.java:31)
   [junit4]   2> 		at org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:142)
   [junit4]   2> 		... 11 more
   [junit4]   2> 	
   [junit4]   2> 540979 T1328 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 540981 T1109 C:control_collection S:shard1 c:collection1 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/pddwr/oq,null}
   [junit4]   2> 541107 T1109 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:35639 35639
   [junit4]   2> 542147 T1110 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:35639 35639
   [junit4]   2> 542148 T1110 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/aliases.json
   [junit4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 		4	/solr/collections/collection1/state.json
   [junit4]   2> 		3	/solr/collections/c8n_1x3_commits/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 		4	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 542149 T1109 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:35644/pddwr/oq
   [junit4]   2> 542149 T1109 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 3 connections to: http://127.0.0.1:41073/pddwr/oq
   [junit4]   2> 542149 T1109 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:36809/pddwr/oq
   [junit4]   2> 542150 T1109 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 13 connections to: http://127.0.0.1:35653/pddwr/oq
   [junit4]   2> 542150 T1109 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:46621/pddwr/oq
   [junit4]   2> 542150 T1109 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:35644/pddwr/oq
   [junit4]   2> 542151 T1109 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:41073/pddwr/oq
   [junit4]   2> 542151 T1109 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:36809/pddwr/oq
   [junit4]   2> 542151 T1109 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:35653/pddwr/oq
   [junit4]   2> 542151 T1109 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:46621/pddwr/oq
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderInitiatedRecoveryOnCommitTest -Dtests.method=test -Dtests.seed=54CB20103163D96E -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=lv_LV -Dtests.timezone=EST -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   84.4s J1 | LeaderInitiatedRecoveryOnCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:35653/pddwr/oq/c8n_1x3_commits_shard1_replica1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([54CB20103163D96E:DC9F1FCA9F9FB496]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 48 more
   [junit4]   2> 542164 T1109 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 54CB20103163D96E-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=DefaultSimilarity, locale=lv_LV, timezone=EST
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=137839720,total=260046848
   [junit4]   2> NOTE: All tests run in this JVM: [ChaosMonkeyNothingIsSafeTest, TestHdfsUpdateLog, TestOmitPositions, TestSurroundQueryParser, TestIndexSearcher, ActionThrottleTest, SignatureUpdateProcessorFactoryTest, TestDistribDocBasedVersion, CoreAdminCreateDiscoverTest, XsltUpdateRequestHandlerTest, TestReplicationHandler, LeaderInitiatedRecoveryOnCommitTest]
   [junit4] Completed on J1 in 84.48s, 1 test, 1 error <<< FAILURES!

[...truncated 1199 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 62 ignored (22 assumptions)

Total time: 66 minutes 19 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 216782796 bytes
Compression is 0.0%
Took 41 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2857 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2857/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:50159/stk/ra/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:50159/stk/ra/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([ACD3A83FB8B1FF39:248797E5164D92C1]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([ACD3A83FB8B1FF39:248797E5164D92C1]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([ACD3A83FB8B1FF39:248797E5164D92C1]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9289 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/init-core-data-001
   [junit4]   2> 90499 T184 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /csu/uk
   [junit4]   2> 90505 T184 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 90506 T185 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 90606 T184 oasc.ZkTestServer.run start zk server on port:57500
   [junit4]   2> 90607 T184 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 90608 T184 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 90615 T192 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@31ccc77f name:ZooKeeperConnection Watcher:127.0.0.1:57500 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 90616 T184 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 90617 T184 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 90617 T184 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 90621 T184 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 90622 T184 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 90624 T195 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4a6717f2 name:ZooKeeperConnection Watcher:127.0.0.1:57500/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 90624 T184 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 90624 T184 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 90625 T184 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 90628 T184 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 90631 T184 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 90634 T184 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 90637 T184 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 90638 T184 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 90642 T184 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 90643 T184 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 90647 T184 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 90647 T184 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 90650 T184 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 90651 T184 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 90654 T184 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 90654 T184 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 90658 T184 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 90658 T184 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 90662 T184 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 90662 T184 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 90666 T184 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 90666 T184 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 90670 T184 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 90670 T184 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 90673 T184 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 90674 T184 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 90677 T184 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 90678 T184 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 91967 T184 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/control-001/cores/collection1
   [junit4]   2> 91972 T184 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 91977 T184 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:57507
   [junit4]   2> 91978 T184 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=57506, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/control-001/cores, hostContext=/csu/uk, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/tempDir-001/control/data}
   [junit4]   2> 91979 T184 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 91979 T184 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/control-001/'
   [junit4]   2> 92026 T184 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/control-001/solr.xml
   [junit4]   2> 92084 T184 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/control-001/cores
   [junit4]   2> 92085 T184 oasc.CoreContainer.<init> New CoreContainer 1985341452
   [junit4]   2> 92085 T184 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/control-001/]
   [junit4]   2> 92086 T184 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/control-001/lib
   [junit4]   2> 92087 T184 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/control-001/lib).
   [junit4]   2> 92101 T184 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 92101 T184 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 92102 T184 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 92102 T184 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 92103 T184 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 92103 T184 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 92103 T184 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 92104 T184 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 92104 T184 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 92105 T184 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 92105 T184 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 92106 T184 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 92107 T184 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 92107 T184 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 92108 T184 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 92108 T184 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:57500/solr
   [junit4]   2> 92108 T184 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 92109 T184 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 92110 T184 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 92112 T207 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1ecd33ab name:ZooKeeperConnection Watcher:127.0.0.1:57500 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 92112 T184 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 92113 T184 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 92116 T184 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 92118 T210 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@24f8d01 name:ZooKeeperConnection Watcher:127.0.0.1:57500/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 92119 T184 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 92121 T184 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 92125 T184 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 92129 T184 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 92134 T184 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 92138 T184 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 92142 T184 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 92146 T184 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 92148 T184 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 92151 T184 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57506_csu%2Fuk
   [junit4]   2> 92152 T184 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:57506_csu%2Fuk
   [junit4]   2> 92155 T184 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 92158 T184 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 92161 T184 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 92164 T184 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:57506_csu%2Fuk
   [junit4]   2> 92165 T184 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 92168 T184 oasc.Overseer.start Overseer (id=93565028541464579-127.0.0.1:57506_csu%2Fuk-n_0000000000) starting
   [junit4]   2> 92172 T184 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 92186 T184 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 92187 T212 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 92188 T184 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 92188 T211 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 92196 T184 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/control-001/cores
   [junit4]   2> 92199 T184 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {dataDir=data/, shard=, config=solrconfig.xml, collection=control_collection, loadOnStartup=true, name=collection1, coreNodeName=, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/control-001/cores/collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/control-001/cores/collection1/, transient=false}
   [junit4]   2> 92200 T184 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/control-001/cores/collection1/
   [junit4]   2> 92200 T184 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 92202 T214 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 92202 T214 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 92204 T210 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 92204 T214 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 92206 T211 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57506/csu/uk",
   [junit4]   2> 	  "node_name":"127.0.0.1:57506_csu%2Fuk",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null} current state version: 0
   [junit4]   2> 92207 T211 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57506/csu/uk",
   [junit4]   2> 	  "node_name":"127.0.0.1:57506_csu%2Fuk",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 92207 T211 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 92208 T211 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 92211 T210 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 93204 T214 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 93205 T214 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 93206 T214 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 93206 T214 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 93208 T214 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 93208 T214 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/control-001/cores/collection1/'
   [junit4]   2> 93228 T214 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 93232 T214 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 93248 T214 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 93274 T214 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 93328 T214 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 93330 T214 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 93345 T214 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 93730 T214 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 93745 T214 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 93748 T214 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 93768 T214 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 93773 T214 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 93777 T214 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 93779 T214 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 93779 T214 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 93779 T214 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 93781 T214 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 93781 T214 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 93781 T214 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 93782 T214 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 93782 T214 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/control-001/cores/collection1/data/
   [junit4]   2> 93783 T214 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@21645be6
   [junit4]   2> 93784 T214 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/control-001/cores/collection1/data
   [junit4]   2> 93784 T214 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/control-001/cores/collection1/data/index/
   [junit4]   2> 93784 T214 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 93784 T214 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/control-001/cores/collection1/data/index
   [junit4]   2> 93785 T214 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=47, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.41625259235176393]
   [junit4]   2> 93786 T214 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@25e4e898 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@57626697),segFN=segments_1,generation=1}
   [junit4]   2> 93786 T214 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 93792 T214 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 93792 T214 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 93792 T214 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 93793 T214 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 93793 T214 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 93793 T214 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 93793 T214 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 93794 T214 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 93794 T214 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 93814 T214 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 93816 T214 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 93818 T214 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 93820 T214 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 93825 T214 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update,/admin/luke,/get,/admin/ping,/schema,/admin/properties,/admin/logging,/replication,/admin/mbeans,/update/json/docs,/admin/file,/admin/threads,/admin/system,standard,/config,/admin/segments,/admin/plugins,/update/json,/update/csv
   [junit4]   2> 93827 T214 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 93828 T214 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 93829 T214 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 93830 T214 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 93830 T214 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 93831 T214 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=27, maxMergeAtOnceExplicit=48, maxMergedSegmentMB=37.3193359375, floorSegmentMB=1.4130859375, forceMergeDeletesPctAllowed=5.289388131669568, segmentsPerTier=30.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 93832 T214 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@25e4e898 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@57626697),segFN=segments_1,generation=1}
   [junit4]   2> 93832 T214 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 93832 T214 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@7c481810[collection1] main
   [junit4]   2> 93833 T214 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 93834 T214 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 93835 T214 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 93835 T214 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 93836 T214 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 93836 T214 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 93837 T214 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 93837 T214 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 93837 T214 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 93838 T214 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 93839 T215 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7c481810[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 93839 T214 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 93840 T218 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:57506/csu/uk collection:control_collection shard:shard1
   [junit4]   2> 93841 T184 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 93842 T184 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 93842 T218 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 93846 T184 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 93847 T184 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 93849 T222 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4d327d98 name:ZooKeeperConnection Watcher:127.0.0.1:57500/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 93849 T184 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 93850 T184 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 93850 T184 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 93851 T218 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 93854 T210 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 93855 T218 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 93855 T218 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C20 name=collection1 org.apache.solr.core.SolrCore@1d3cff1c url=http://127.0.0.1:57506/csu/uk/collection1 node=127.0.0.1:57506_csu%2Fuk C20_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:57506/csu/uk, node_name=127.0.0.1:57506_csu%2Fuk, core=collection1, state=down}
   [junit4]   2> 93855 T218 C:control_collection S:shard1 C20 P57506 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:57506/csu/uk/collection1/
   [junit4]   2> 93856 T218 C:control_collection S:shard1 C20 P57506 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 93856 T211 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 93856 T218 C:control_collection S:shard1 C20 P57506 oasc.SyncStrategy.syncToMe http://127.0.0.1:57506/csu/uk/collection1/ has no replicas
   [junit4]   2> 93856 T184 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 93857 T218 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:57506/csu/uk/collection1/ shard1
   [junit4]   2> 93857 T184 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 93857 T218 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 93858 T184 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 93858 T222 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 93858 T210 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 93859 T184 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 93861 T225 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@d27497e name:ZooKeeperConnection Watcher:127.0.0.1:57500/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 93862 T184 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 93863 T184 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 93868 T211 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 93868 T211 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 93868 T210 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 93873 T211 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57506/csu/uk",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 93874 T211 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 93876 T222 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 93876 T210 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 93883 T226 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 93885 T226 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 93885 T227 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 93920 T218 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:57506/csu/uk/collection1/ and leader is http://127.0.0.1:57506/csu/uk/collection1/
   [junit4]   2> 93921 T218 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:57506/csu/uk
   [junit4]   2> 93921 T218 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 93921 T218 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 93921 T218 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 93923 T226 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 93924 T211 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57506/csu/uk",
   [junit4]   2> 	  "node_name":"127.0.0.1:57506_csu%2Fuk",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1"} current state version: 4
   [junit4]   2> 93926 T211 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57506/csu/uk",
   [junit4]   2> 	  "node_name":"127.0.0.1:57506_csu%2Fuk",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1"}
   [junit4]   2> 94031 T226 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 94031 T227 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 95620 T184 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/shard-1-001/cores/collection1
   [junit4]   2> 95624 T184 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/shard-1-001
   [junit4]   2> 95625 T184 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 95628 T184 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:57523
   [junit4]   2> 95628 T184 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=57522, solrconfig=solrconfig.xml, hostContext=/csu/uk, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/tempDir-001/jetty1, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/shard-1-001/cores}
   [junit4]   2> 95629 T184 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 95630 T184 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/shard-1-001/'
   [junit4]   2> 95676 T184 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/shard-1-001/solr.xml
   [junit4]   2> 95731 T184 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/shard-1-001/cores
   [junit4]   2> 95732 T184 oasc.CoreContainer.<init> New CoreContainer 1133049693
   [junit4]   2> 95733 T184 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/shard-1-001/]
   [junit4]   2> 95733 T184 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/shard-1-001/lib
   [junit4]   2> 95734 T184 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/shard-1-001/lib).
   [junit4]   2> 95748 T184 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 95749 T184 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 95749 T184 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 95749 T184 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 95750 T184 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 95750 T184 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 95751 T184 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 95751 T184 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 95752 T184 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 95752 T184 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 95752 T184 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 95753 T184 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 95754 T184 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 95754 T184 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 95755 T184 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 95755 T184 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:57500/solr
   [junit4]   2> 95756 T184 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 95756 T184 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 95758 T184 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 95760 T239 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@18ead462 name:ZooKeeperConnection Watcher:127.0.0.1:57500 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 95761 T184 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 95762 T184 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 95765 T184 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 95767 T242 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@19ec0c95 name:ZooKeeperConnection Watcher:127.0.0.1:57500/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 95767 T184 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 95776 T184 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 96787 T184 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57522_csu%2Fuk
   [junit4]   2> 96788 T184 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:57522_csu%2Fuk
   [junit4]   2> 96793 T184 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 96796 T184 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/shard-1-001/cores
   [junit4]   2> 96798 T184 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {transient=false, loadOnStartup=true, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/shard-1-001/cores/collection1/, schema=schema.xml, dataDir=data/, config=solrconfig.xml, shard=, name=collection1, coreNodeName=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/shard-1-001/cores/collection1, collection=collection1}
   [junit4]   2> 96799 T184 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/shard-1-001/cores/collection1/
   [junit4]   2> 96799 T184 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 96803 T243 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 96804 T243 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 96807 T226 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 96808 T243 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 96808 T211 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57522/csu/uk",
   [junit4]   2> 	  "node_name":"127.0.0.1:57522_csu%2Fuk",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null} current state version: 5
   [junit4]   2> 96808 T243 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 96809 T211 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57522/csu/uk",
   [junit4]   2> 	  "node_name":"127.0.0.1:57522_csu%2Fuk",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 96810 T211 oasco.ReplicaMutator.updateState Collection already exists with numShards=3
   [junit4]   2> 96810 T243 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 96810 T211 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard3
   [junit4]   2> 96813 T211 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 96819 T243 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 96819 T243 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 96820 T243 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 96820 T243 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 96821 T243 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 96821 T243 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 96823 T243 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 96823 T243 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/shard-1-001/cores/collection1/'
   [junit4]   2> 96841 T243 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 96844 T243 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 96858 T243 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 96884 T243 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 96958 T243 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 96960 T243 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 96972 T243 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 97401 T243 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 97422 T243 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 97427 T243 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 97450 T243 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 97457 T243 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 97463 T243 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 97465 T243 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 97466 T243 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 97466 T243 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 97467 T243 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 97468 T243 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 97468 T243 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 97469 T243 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 97469 T243 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 97470 T243 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@21645be6
   [junit4]   2> 97471 T243 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/shard-1-001/cores/collection1/data
   [junit4]   2> 97472 T243 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 97472 T243 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 97472 T243 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest ACD3A83FB8B1FF39-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 97473 T243 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=47, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.41625259235176393]
   [junit4]   2> 97475 T243 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@574154a4 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@21c07bd),segFN=segments_1,generation=1}
   [junit4]   2> 97475 T243 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 97486 T243 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 97487 T243 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 97487 T243 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 97488 T243 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 97488 T243 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 97489 T243 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 97490 T243 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 97491 T243 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 97491 T243 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 97518 T243 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 97529 T243 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 97533 T243 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 97539 T243 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 97553 T243 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update,/admin/luke,/get,/admin/ping,/schema,/admin/properties,/admin/logging,/replication,/admin/mbeans,/update/json/docs,/admin/file,/admin/threads,/admin/system,standard,/config,/admin/segments,/admin/plugins,/update/json,/update/csv
   [junit4]   2> 97556 T243 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 97559 T243 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 97559 T243 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 97561 T243 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 97561 T243 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 97563 T243 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=27, maxMergeAtOnceExplicit=48, maxMergedSegmentMB=37.3193359375, floorSegmentMB=1.4130859375, forceMergeDeletesPctAllowed=5.289388131669568, segmentsPerTier=30.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 97564 T243 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@574154a4 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@21c07bd),segFN=segments_1,generation=1}
   [junit4]   2> 97564 T243 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 97565 T243 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@544187a0[collection1] main
   [junit4]   2> 97565 T243 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 97567 T243 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 97568 T243 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 97568 T243 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 97583 T243 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 97583 T243 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 97585 T243 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 97585 T243 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 97585 T243 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 97586 T243 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 97588 T244 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@544187a0[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 97589 T243 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 97590 T247 C:collection1 S:shard3 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:57522/csu/uk collection:collection1 shard:shard3
   [junit4]   2> 97591 T184 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 97591 T184 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 97592 T247 C:collection1 S:shard3 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard3/election
   [junit4]   2> 97606 T247 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard3
   [junit4]   2> 97610 T226 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 97611 T247 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 97611 T247 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C21 name=collection1 org.apache.solr.core.SolrCore@2007b218 url=http://127.0.0.1:57522/csu/uk/collection1 node=127.0.0.1:57522_csu%2Fuk C21_STATE=coll:collection1 core:collection1 props:{base_url=http://127.0.0.1:57522/csu/uk, node_name=127.0.0.1:57522_csu%2Fuk, core=collection1, state=down}
   [junit4]   2> 97612 T247 C:collection1 S:shard3 C21 P57522 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:57522/csu/uk/collection1/
   [junit4]   2> 97612 T247 C:collection1 S:shard3 C21 P57522 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 97613 T247 C:collection1 S:shard3 C21 P57522 oasc.SyncStrategy.syncToMe http://127.0.0.1:57522/csu/uk/collection1/ has no replicas
   [junit4]   2> 97613 T211 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 97613 T247 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:57522/csu/uk/collection1/ shard3
   [junit4]   2> 97614 T247 C:collection1 S:shard3 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard3
   [junit4]   2> 97616 T211 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 97618 T242 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 97622 T242 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 97628 T211 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57522/csu/uk",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 97630 T211 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 97631 T242 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 97633 T242 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 97636 T226 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 97676 T247 C:collection1 S:shard3 oasc.ZkController.register We are http://127.0.0.1:57522/csu/uk/collection1/ and leader is http://127.0.0.1:57522/csu/uk/collection1/
   [junit4]   2> 97677 T247 C:collection1 S:shard3 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:57522/csu/uk
   [junit4]   2> 97677 T247 C:collection1 S:shard3 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 97677 T247 C:collection1 S:shard3 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 97678 T247 C:collection1 S:shard3 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 97680 T226 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 97682 T211 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57522/csu/uk",
   [junit4]   2> 	  "node_name":"127.0.0.1:57522_csu%2Fuk",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard3"} current state version: 5
   [junit4]   2> 97683 T211 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57522/csu/uk",
   [junit4]   2> 	  "node_name":"127.0.0.1:57522_csu%

[...truncated too long message...]

707 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:17472 17472
   [junit4]   2> 1724654 T5708 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:17472 17472
   [junit4]   2> 1724655 T5708 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 		2	/solr/collections/collection1/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1724656 T5707 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:34193/uz/i
   [junit4]   2> 1724656 T5707 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:36087/uz/i
   [junit4]   2> 1724656 T5707 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:47010/uz/i
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=ACD3A83FB8B1FF39 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=sr_ME -Dtests.timezone=Etc/GMT-5 -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   67.1s J1 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([ACD3A83FB8B1FF39:248797E5164D92C1]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 1724676 T5707 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest ACD3A83FB8B1FF39-001
   [junit4]   2> 67103 T5706 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=DummyCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=DUMMY, chunkSize=6, maxDocsPerChunk=625, blockSize=6), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=DUMMY, chunkSize=6, blockSize=6)), sim=DefaultSimilarity, locale=sr_ME, timezone=Etc/GMT-5
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=151137352,total=343932928
   [junit4]   2> NOTE: All tests run in this JVM: [SharedFSAutoReplicaFailoverTest, DirectUpdateHandlerOptimizeTest, TestFiltering, SSLMigrationTest, RankQueryTest, StressHdfsTest, TestStandardQParsers, DateFieldTest, TestExceedMaxTermLength, PathHierarchyTokenizerFactoryTest, TestReqParamsAPI, AnalysisAfterCoreReloadTest, DistributedQueryElevationComponentTest, TestSolrConfigHandlerCloud, NotRequiredUniqueKeyTest, TestFaceting, ExternalCollectionsTest, TestDistributedGrouping, TestExpandComponent, SampleTest, SolrCmdDistributorTest, QueryEqualityTest, TestBinaryResponseWriter, OverriddenZkACLAndCredentialsProvidersTest, TestSolrCoreProperties, TestFileDictionaryLookup, LeaderInitiatedRecoveryOnCommitTest, MigrateRouteKeyTest, TestFuzzyAnalyzedSuggestions, RegexBoostProcessorTest, CoreAdminCreateDiscoverTest, SolrTestCaseJ4Test, PluginInfoTest, ChaosMonkeyNothingIsSafeTest, NoCacheHeaderTest, TestCollapseQParserPlugin, TestWriterPerf, TestSurroundQueryParser, UpdateRequestProcessorFactoryTest, TestInitParams, TestOverriddenPrefixQueryForCustomFieldType, CloudMLTQParserTest, InfoHandlerTest, TestJsonRequest, SuggestComponentTest, TermsComponentTest, ZkCLITest, TestQuerySenderNoQuery, TestDynamicLoading, TestSuggestSpellingConverter, CoreMergeIndexesAdminHandlerTest, TestCodecSupport, TestConfigReload, TestSchemaVersionResource, TestValueSourceCache, ActionThrottleTest, ChangedSchemaMergeTest, TestSerializedLuceneMatchVersion, TestCloudManagedSchemaConcurrent, ShardSplitTest, TestReplicationHandler, TestCloudManagedSchema, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J1 in 68.15s, 1 test, 1 error <<< FAILURES!

[...truncated 703 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 62 ignored (22 assumptions)

Total time: 59 minutes 25 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 211696231 bytes
Compression is 0.0%
Took 44 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2856 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2856/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:24613/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:24613/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([6B7571477A4A303:8EE368CED958CEFB]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:32723/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:32723/collection1
	at __randomizedtesting.SeedInfo.seed([6B7571477A4A303:8EE368CED958CEFB]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:32723/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:24761/repfacttest_c8n_1x3_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:24761/repfacttest_c8n_1x3_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([6B7571477A4A303:8EE368CED958CEFB]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:24761/repfacttest_c8n_1x3_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9633 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/init-core-data-001
   [junit4]   2> 648841 T1466 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 648846 T1466 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 648847 T1467 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 648947 T1466 oasc.ZkTestServer.run start zk server on port:56072
   [junit4]   2> 648948 T1466 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 648949 T1466 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 648952 T1474 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5efd2ab2 name:ZooKeeperConnection Watcher:127.0.0.1:56072 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 648952 T1466 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 648953 T1466 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 648953 T1466 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 648955 T1466 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 648956 T1466 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 648957 T1477 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1e9a3a87 name:ZooKeeperConnection Watcher:127.0.0.1:56072/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 648957 T1466 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 648957 T1466 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 648958 T1466 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 648959 T1466 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 648960 T1466 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 648961 T1466 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 648962 T1466 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 648963 T1466 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 648964 T1466 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 648965 T1466 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 649067 T1466 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 649067 T1466 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 649069 T1466 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 649069 T1466 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 649071 T1466 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 649071 T1466 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 649073 T1466 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 649073 T1466 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 649075 T1466 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 649075 T1466 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 649077 T1466 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 649077 T1466 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 649078 T1466 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 649079 T1466 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 649080 T1466 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 649081 T1466 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 649082 T1466 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 649083 T1466 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 650570 T1466 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/control-001/cores/collection1
   [junit4]   2> 650574 T1466 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 650577 T1466 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:14069
   [junit4]   2> 650577 T1466 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/control-001/cores, hostPort=37156, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/tempDir-001/control/data}
   [junit4]   2> 650578 T1466 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 650578 T1466 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/control-001/'
   [junit4]   2> 650620 T1466 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/control-001/solr.xml
   [junit4]   2> 650677 T1466 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/control-001/cores
   [junit4]   2> 650677 T1466 oasc.CoreContainer.<init> New CoreContainer 287801376
   [junit4]   2> 650677 T1466 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/control-001/]
   [junit4]   2> 650678 T1466 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/control-001/lib
   [junit4]   2> 650678 T1466 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/control-001/lib).
   [junit4]   2> 650687 T1466 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 650687 T1466 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 650687 T1466 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 650688 T1466 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 650688 T1466 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 650688 T1466 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 650688 T1466 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 650689 T1466 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 650689 T1466 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 650689 T1466 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 650689 T1466 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 650690 T1466 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 650690 T1466 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 650690 T1466 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 650690 T1466 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 650691 T1466 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:56072/solr
   [junit4]   2> 650691 T1466 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 650691 T1466 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 650692 T1466 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 650693 T1489 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7fdcd47c name:ZooKeeperConnection Watcher:127.0.0.1:56072 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 650694 T1466 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 650695 T1466 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 650696 T1466 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 650697 T1492 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@46ec2881 name:ZooKeeperConnection Watcher:127.0.0.1:56072/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 650698 T1466 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 650699 T1466 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 650701 T1466 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 650703 T1466 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 650704 T1466 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 650706 T1466 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 650707 T1466 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 650709 T1466 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 650710 T1466 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 650711 T1466 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37156_
   [junit4]   2> 650711 T1466 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:37156_
   [junit4]   2> 650712 T1466 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 650714 T1466 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 650715 T1466 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 650716 T1466 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:37156_
   [junit4]   2> 650716 T1466 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 650718 T1466 oasc.Overseer.start Overseer (id=93564118232334339-127.0.0.1:37156_-n_0000000000) starting
   [junit4]   2> 650719 T1466 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 650724 T1466 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 650725 T1494 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 650725 T1466 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 650725 T1493 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 650729 T1466 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/control-001/cores
   [junit4]   2> 650730 T1466 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/control-001/cores/collection1/, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/control-001/cores/collection1, name=collection1, config=solrconfig.xml, coreNodeName=, loadOnStartup=true, collection=control_collection, dataDir=data/, shard=, schema=schema.xml}
   [junit4]   2> 650730 T1466 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/control-001/cores/collection1/
   [junit4]   2> 650731 T1466 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 650731 T1496 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 650732 T1496 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 650732 T1492 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 650732 T1496 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 650733 T1493 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37156",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:37156_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null} current state version: 0
   [junit4]   2> 650734 T1493 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37156",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:37156_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 650734 T1493 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 650734 T1493 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 650735 T1492 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 651733 T1496 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 651734 T1496 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 651734 T1496 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 651734 T1496 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 651735 T1496 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 651735 T1496 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/control-001/cores/collection1/'
   [junit4]   2> 651748 T1496 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 651749 T1496 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 651757 T1496 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 651772 T1496 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 651813 T1496 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 651914 T1496 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 651921 T1496 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 652187 T1496 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 652198 T1496 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 652200 T1496 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 652208 T1496 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 652211 T1496 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 652214 T1496 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 652214 T1496 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 652215 T1496 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 652215 T1496 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 652215 T1496 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 652216 T1496 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 652216 T1496 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 652216 T1496 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 652216 T1496 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/control-001/cores/collection1/data/
   [junit4]   2> 652216 T1496 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@709e8c34
   [junit4]   2> 652217 T1496 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/control-001/cores/collection1/data
   [junit4]   2> 652217 T1496 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/control-001/cores/collection1/data/index/
   [junit4]   2> 652217 T1496 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 652218 T1496 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/control-001/cores/collection1/data/index
   [junit4]   2> 652218 T1496 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=17, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.13412214211182302]
   [junit4]   2> 652219 T1496 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@233ed18 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1f12c3c9),segFN=segments_1,generation=1}
   [junit4]   2> 652219 T1496 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 652223 T1496 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 652223 T1496 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 652223 T1496 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 652223 T1496 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 652224 T1496 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 652224 T1496 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 652224 T1496 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 652224 T1496 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 652224 T1496 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 652237 T1496 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 652238 T1496 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 652239 T1496 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 652241 T1496 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 652245 T1496 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /config,/admin/system,/admin/file,/admin/threads,/admin/plugins,/admin/properties,/schema,/admin/ping,/update,/admin/luke,/update/json,/replication,/admin/mbeans,standard,/update/csv,/get,/admin/logging,/update/json/docs,/admin/segments
   [junit4]   2> 652246 T1496 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 652247 T1496 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 652247 T1496 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 652248 T1496 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 652248 T1496 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 652249 T1496 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=49, maxMergeAtOnceExplicit=48, maxMergedSegmentMB=45.916015625, floorSegmentMB=0.916015625, forceMergeDeletesPctAllowed=3.154167937680401, segmentsPerTier=44.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3414021095843023
   [junit4]   2> 652249 T1496 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@233ed18 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1f12c3c9),segFN=segments_1,generation=1}
   [junit4]   2> 652249 T1496 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 652250 T1496 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@4b1b558f[collection1] main
   [junit4]   2> 652250 T1496 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 652251 T1496 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 652251 T1496 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 652251 T1496 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 652251 T1496 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 652252 T1496 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 652252 T1496 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 652252 T1496 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 652252 T1496 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 652253 T1496 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 652253 T1497 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4b1b558f[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 652254 T1496 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 652255 T1500 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:37156 collection:control_collection shard:shard1
   [junit4]   2> 652255 T1466 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 652255 T1466 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 652255 T1500 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 652258 T1466 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 652259 T1466 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 652260 T1500 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 652260 T1504 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6822108a name:ZooKeeperConnection Watcher:127.0.0.1:56072/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 652261 T1466 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 652261 T1492 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 652261 T1466 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 652261 T1500 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 652262 T1500 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 652262 T1466 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> ASYNC  NEW_CORE C574 name=collection1 org.apache.solr.core.SolrCore@7d40f297 url=http://127.0.0.1:37156/collection1 node=127.0.0.1:37156_ C574_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:37156, state=down, node_name=127.0.0.1:37156_, core=collection1}
   [junit4]   2> 652262 T1500 C:control_collection S:shard1 C574 P37156 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:37156/collection1/
   [junit4]   2> 652262 T1493 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 652264 T1500 C:control_collection S:shard1 C574 P37156 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 652264 T1500 C:control_collection S:shard1 C574 P37156 oasc.SyncStrategy.syncToMe http://127.0.0.1:37156/collection1/ has no replicas
   [junit4]   2> 652264 T1500 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:37156/collection1/ shard1
   [junit4]   2> 652264 T1500 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 652265 T1492 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 652265 T1504 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 652266 T1466 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 652268 T1492 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 652269 T1493 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37156",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 652372 T1504 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 652372 T1492 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 652421 T1500 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:37156/collection1/ and leader is http://127.0.0.1:37156/collection1/
   [junit4]   2> 652422 T1500 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:37156
   [junit4]   2> 652422 T1500 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 652422 T1500 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 652422 T1500 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 652423 T1492 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 652424 T1493 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37156",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:37156_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null} current state version: 3
   [junit4]   2> 652424 T1493 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37156",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:37156_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 652527 T1492 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 652527 T1504 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 653857 T1466 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/shard-1-001/cores/collection1
   [junit4]   2> 653861 T1466 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/shard-1-001
   [junit4]   2> 653862 T1466 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 653865 T1466 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:25077
   [junit4]   2> 653865 T1466 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, hostPort=32723, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/shard-1-001/cores, solrconfig=solrconfig.xml, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/tempDir-001/jetty1}
   [junit4]   2> 653866 T1466 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 653866 T1466 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/shard-1-001/'
   [junit4]   2> 653925 T1466 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/shard-1-001/solr.xml
   [junit4]   2> 653998 T1466 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/shard-1-001/cores
   [junit4]   2> 653998 T1466 oasc.CoreContainer.<init> New CoreContainer 885921915
   [junit4]   2> 653999 T1466 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/shard-1-001/]
   [junit4]   2> 653999 T1466 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/shard-1-001/lib
   [junit4]   2> 654000 T1466 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/shard-1-001/lib).
   [junit4]   2> 654014 T1466 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 654014 T1466 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 654015 T1466 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 654015 T1466 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 654015 T1466 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 654016 T1466 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 654016 T1466 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 654016 T1466 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 654017 T1466 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 654017 T1466 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 654017 T1466 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 654018 T1466 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 654018 T1466 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 654019 T1466 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 654019 T1466 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 654019 T1466 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:56072/solr
   [junit4]   2> 654020 T1466 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 654020 T1466 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 654021 T1466 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 654023 T1516 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@47e4d142 name:ZooKeeperConnection Watcher:127.0.0.1:56072 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 654023 T1466 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 654024 T1466 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 654026 T1466 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 654028 T1519 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2636e0b0 name:ZooKeeperConnection Watcher:127.0.0.1:56072/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 654028 T1466 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 654033 T1466 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 655036 T1466 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:32723_
   [junit4]   2> 655037 T1466 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:32723_
   [junit4]   2> 655040 T1466 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 655042 T1466 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/shard-1-001/cores
   [junit4]   2> 655044 T1466 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/shard-1-001/cores/collection1, shard=, collection=collection1, dataDir=data/, coreNodeName=, loadOnStartup=true, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/shard-1-001/cores/collection1/, name=collection1, schema=schema.xml, transient=false}
   [junit4]   2> 655044 T1466 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/shard-1-001/cores/collection1/
   [junit4]   2> 655044 T1466 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 655045 T1520 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 655045 T1520 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 655046 T1492 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 655046 T1520 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 655047 T1493 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32723",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:32723_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null} current state version: 4
   [junit4]   2> 655047 T1493 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32723",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:32723_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 655048 T1493 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 655048 T1493 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 655049 T1519 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 655049 T1492 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 655049 T1504 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 656047 T1520 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 656048 T1520 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 656048 T1520 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 656049 T1520 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 656049 T1520 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 656050 T1520 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/shard-1-001/cores/collection1/'
   [junit4]   2> 656067 T1520 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 656069 T1520 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 656080 T1520 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 656102 T1520 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 656150 T1520 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 656251 T1520 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 656261 T1520 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 656574 T1520 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 656585 T1520 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 656587 T1520 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 656595 T1520 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 656599 T1520 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 656602 T1520 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 656602 T1520 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 656602 T1520 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 656603 T1520 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 656603 T1520 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 656603 T1520 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 656604 T1520 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 656604 T1520 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 656604 T1520 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 656604 T1520 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@709e8c34
   [junit4]   2> 656605 T1520 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/shard-1-001/cores/collection1/data
   [junit4]   2> 656605 T1520 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 656605 T1520 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 656606 T1520 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 6B7571477A4A303-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 656606 T1520 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=17, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.13412214211182302]
   [junit4]   2> 656607 T1520 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@76480bfa lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@740a86c0),segFN=segments_1,generation=1}
   [junit4]   2> 656607 T1520 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 656612 T1520 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 656612 T1520 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 656612 T1520 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 656612 T1520 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 656612 T1520 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 656613 T1520 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 656613 T1520 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 656613 T1520 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 656613 T1520 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 656626 T1520 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 656627 T1520 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 656629 T1520 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 656630 T1520 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 656636 T1520 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /config,/admin/system,/admin/file,/admin/threads,/admin/plugins,/admin/properties,/schema,/admin/ping,/update,/admin/luke,/update/json,/replication,/admin/mbeans,standard,/update/csv,/get,/admin/logging,/update/json/docs,/admin/segments
   [junit4]   2> 656638 T1520 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 656639 T1520 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 656639 T1520 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 656640 T1520 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 656640 T1520 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 656641 T1520 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=49, maxMergeAtOnceExplicit=48, maxMergedSegmentMB=45.916015625, floorSegmentMB=0.916015625, forceMergeDeletesPctAllowed=3.154167937680401, segmentsPerTier=44.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3414021095843023
   [junit4]   2> 656642 T1520 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@76480bfa lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@740a86c0),segFN=segments_1,generation=1}
   [junit4]   2> 656642 T1520 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 656642 T1520 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@16974c20[collection1] main
   [junit4]   2> 656642 T1520 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 656643 T1520 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 656643 T1520 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 656643 T1520 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 656644 T1520 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 656644 T1520 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 656644 T1520 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 656644 T1520 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 656644 T1520 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 656645 T1520 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 656646 T1521 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@16974c20[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 656646 T1520 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 656647 T1524 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:32723 collection:collection1 shard:shard1
   [junit4]   2> 656647 T1466 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 656647 T1466 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 656648 T1524 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 656652 T1524 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 656653 T1492 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 656654 T1524 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 656654 T1524 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C575 name=collection1 org.apache.solr.core.SolrCore@51b6b699 url=http://127.0.0.1:32723/collection1 node=127.0.0.1:32723_ C575_STATE=coll:collection1 core:collection1 props:{base_url=http://127.0.0.1:32723, state=down, node_name=127.0.0.1:32723_, core=collection1}
   [junit4]   2> 656654 T1524 C:collection1 S:shard1 C575 P32723 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:32723/collection1/
   [junit4]   2> 656654 T1493 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 656654 T1524 C:collection1 S:shard1 C575 P32723 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 656655 T1524 C:collection1 S:shard1 C575 P32723 oasc.SyncStrategy.syncToMe http://127.0.0.1:32723/collection1/ has no replicas
   [junit4]   2> 656655 T1524 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:32723/collection1/ shard1
   [junit4]   2> 656655 T1524 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 656655 T1492 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 656656 T1519 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 656656 T1504 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 656659 T1492 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 656660 T1493 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32723",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 656763 T1504 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 656763 T1519 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 656763 T1492 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 656812 T1524 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:32723/collection1/ and leader is http://127.0.0.1:32723/collection1/
   [junit4]   2> 656813 T1524 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:32723
   [junit4]   2> 656813 T1524 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 656813 T1524 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 656813 T1524 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 656814 T1492 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 656815 T1493 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32723",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:32723_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null} current state version: 7
   [junit4]   2> 656815 T1493 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32723",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:32723_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 656918 T1519 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 656918 T1492 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clu

[...truncated too long message...]

ections to: http://127.0.0.1:24761/
   [junit4]   2> 1601715 T3751 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 2 connections to: http://127.0.0.1:24744/
   [junit4]   2> 1601715 T3751 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 3 connections to: http://127.0.0.1:24714/
   [junit4]   2> 1601716 T3751 C:control_collection S:shard1 c:collection1 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 1601716 T3751 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:24752/
   [junit4]   2> 1601716 T3751 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:24761/
   [junit4]   2> 1601717 T3751 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:24744/
   [junit4]   2> 1601717 T3751 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:24714/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=6B7571477A4A303 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=sv -Dtests.timezone=CST6CDT -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   75.1s J1 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:24761/repfacttest_c8n_1x3_shard1_replica3
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([6B7571477A4A303:8EE368CED958CEFB]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:24761/repfacttest_c8n_1x3_shard1_replica3
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 48 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 53 more
   [junit4]   2> 1601748 T3751 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 6B7571477A4A303-001
   [junit4]   2> 75127 T3750 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {timestamp=Lucene50(blocksize=128), text=PostingsFormat(name=Memory doPackFST= true), id=PostingsFormat(name=LuceneVarGapFixedInterval), intDefault=PostingsFormat(name=Asserting), range_facet_l=Lucene50(blocksize=128), a_t=Lucene50(blocksize=128), _version_=PostingsFormat(name=Asserting), multiDefault=Lucene50(blocksize=128)}, docValues:{timestamp=DocValuesFormat(name=Memory)}, sim=DefaultSimilarity, locale=sv, timezone=CST6CDT
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=127753656,total=343408640
   [junit4]   2> NOTE: All tests run in this JVM: [BasicFunctionalityTest, TestFiltering, TestManagedSchemaDynamicFieldResource, TestSort, TestHdfsUpdateLog, DeleteLastCustomShardedReplicaTest, IndexBasedSpellCheckerTest, TestJoin, TestReplicationHandler, OutOfBoxZkACLAndCredentialsProvidersTest, TestClassNameShortening, TestOmitPositions, AnalysisErrorHandlingTest, TestFaceting, DistributedFacetPivotSmallTest, TestCSVLoader, SolrIndexSplitterTest, DistributedSuggestComponentTest, ShardRoutingTest, TestWordDelimiterFilterFactory, AutoCommitTest, StatelessScriptUpdateProcessorFactoryTest, TestCollationFieldDocValues, DistributedFacetPivotSmallAdvancedTest, TestFastOutputStream, TestStressRecovery, TriLevelCompositeIdRoutingTest, OutputWriterTest, TestDefaultStatsCache, TestElisionMultitermQuery, TestReqParamsAPI, DistributedFacetPivotLargeTest, ConnectionManagerTest, TestManagedSynonymFilterFactory, TestSolrConfigHandlerCloud, TestShardHandlerFactory, CursorMarkTest, LeaderFailoverAfterPartitionTest, InfoHandlerTest, TestGroupingSearch, PrimUtilsTest, TestCloudPivotFacet, TestClusterStateMutator, BadIndexSchemaTest, UpdateParamsTest, StatsComponentTest, ZkStateWriterTest, TestQueryTypes, DebugComponentTest, TestCSVResponseWriter, BadCopyFieldTest, CopyFieldTest, TestCollapseQParserPlugin, CoreAdminRequestStatusTest, TestXIncludeConfig, ClusterStateTest, ChaosMonkeyNothingIsSafeTest, TestCloudSchemaless, UnloadDistributedZkTest, ReplicationFactorTest]
   [junit4] Completed on J1 in 75.66s, 1 test, 1 error <<< FAILURES!

[...truncated 722 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 62 ignored (22 assumptions)

Total time: 54 minutes 1 second
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 210287386 bytes
Compression is 0.0%
Took 40 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2855 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2855/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:48527/re/ad/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48527/re/ad/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([2EFCD7E4519914F5:A6A8E83EFF65790D]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28025/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28025/collection1
	at __randomizedtesting.SeedInfo.seed([2EFCD7E4519914F5:A6A8E83EFF65790D]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28025/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48738/repfacttest_c8n_1x3_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48738/repfacttest_c8n_1x3_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([2EFCD7E4519914F5:A6A8E83EFF65790D]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48738/repfacttest_c8n_1x3_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9959 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/init-core-data-001
   [junit4]   2> 1230927 T2454 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 1230932 T2454 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1230933 T2455 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1231033 T2454 oasc.ZkTestServer.run start zk server on port:12062
   [junit4]   2> 1231034 T2454 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1231034 T2454 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1231038 T2462 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@16d8b765 name:ZooKeeperConnection Watcher:127.0.0.1:12062 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1231038 T2454 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1231039 T2454 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1231039 T2454 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1231043 T2454 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1231044 T2454 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1231045 T2465 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@128dd8fb name:ZooKeeperConnection Watcher:127.0.0.1:12062/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1231046 T2454 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1231046 T2454 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1231046 T2454 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1231049 T2454 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1231051 T2454 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1231052 T2454 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1231054 T2454 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1231055 T2454 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1231058 T2454 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1231059 T2454 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1231161 T2454 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1231162 T2454 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1231163 T2454 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1231164 T2454 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1231166 T2454 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1231166 T2454 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1231168 T2454 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1231169 T2454 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1231170 T2454 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1231171 T2454 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1231173 T2454 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1231174 T2454 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1231176 T2454 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1231176 T2454 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1231178 T2454 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1231179 T2454 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1231181 T2454 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1231181 T2454 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1232656 T2454 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/control-001/cores/collection1
   [junit4]   2> 1232661 T2454 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1232667 T2454 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:59347
   [junit4]   2> 1232667 T2454 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=64567, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/control-001/cores, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/tempDir-001/control/data}
   [junit4]   2> 1232668 T2454 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 1232668 T2454 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/control-001/'
   [junit4]   2> 1232712 T2454 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/control-001/solr.xml
   [junit4]   2> 1232765 T2454 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/control-001/cores
   [junit4]   2> 1232766 T2454 oasc.CoreContainer.<init> New CoreContainer 224568597
   [junit4]   2> 1232766 T2454 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/control-001/]
   [junit4]   2> 1232767 T2454 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/control-001/lib
   [junit4]   2> 1232767 T2454 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/control-001/lib).
   [junit4]   2> 1232780 T2454 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1232780 T2454 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1232781 T2454 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1232781 T2454 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1232781 T2454 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1232782 T2454 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1232782 T2454 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1232782 T2454 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1232783 T2454 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1232783 T2454 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1232783 T2454 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1232784 T2454 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1232784 T2454 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1232785 T2454 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1232785 T2454 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1232785 T2454 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:12062/solr
   [junit4]   2> 1232786 T2454 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1232786 T2454 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1232787 T2454 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1232788 T2477 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4f5ea9e5 name:ZooKeeperConnection Watcher:127.0.0.1:12062 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1232789 T2454 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1232790 T2454 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1232792 T2454 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1232793 T2480 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7f2d1e53 name:ZooKeeperConnection Watcher:127.0.0.1:12062/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1232793 T2454 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1232795 T2454 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1232797 T2454 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1232800 T2454 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1232802 T2454 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1232804 T2454 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1232807 T2454 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1232809 T2454 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1232810 T2454 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1232812 T2454 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:64567_
   [junit4]   2> 1232812 T2454 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:64567_
   [junit4]   2> 1232814 T2454 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1232816 T2454 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1232817 T2454 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1232819 T2454 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:64567_
   [junit4]   2> 1232819 T2454 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1232821 T2454 oasc.Overseer.start Overseer (id=93563512902582275-127.0.0.1:64567_-n_0000000000) starting
   [junit4]   2> 1232823 T2454 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1232830 T2454 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 1232831 T2482 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1232831 T2454 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1232831 T2481 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1232835 T2454 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/control-001/cores
   [junit4]   2> 1232837 T2454 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {loadOnStartup=true, shard=, collection=control_collection, schema=schema.xml, dataDir=data/, coreNodeName=, name=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/control-001/cores/collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/control-001/cores/collection1/, config=solrconfig.xml, transient=false}
   [junit4]   2> 1232838 T2454 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/control-001/cores/collection1/
   [junit4]   2> 1232838 T2454 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1232839 T2484 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1232839 T2484 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1232840 T2480 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1232840 T2484 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1232841 T2481 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:64567",
   [junit4]   2> 	  "node_name":"127.0.0.1:64567_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection"} current state version: 0
   [junit4]   2> 1232841 T2481 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:64567",
   [junit4]   2> 	  "node_name":"127.0.0.1:64567_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection"}
   [junit4]   2> 1232842 T2481 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 1232842 T2481 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1232843 T2480 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1233841 T2484 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1233842 T2484 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1233842 T2484 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1233842 T2484 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1233843 T2484 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1233843 T2484 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/control-001/cores/collection1/'
   [junit4]   2> 1233856 T2484 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1233858 T2484 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1233865 T2484 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1233880 T2484 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 1233924 T2484 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1234025 T2484 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1234032 T2484 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1234323 T2484 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1234338 T2484 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1234341 T2484 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1234352 T2484 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1234356 T2484 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1234361 T2484 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1234362 T2484 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1234362 T2484 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1234362 T2484 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1234363 T2484 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1234363 T2484 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1234364 T2484 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1234364 T2484 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1234364 T2484 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/control-001/cores/collection1/data/
   [junit4]   2> 1234365 T2484 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@21018702
   [junit4]   2> 1234365 T2484 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/control-001/cores/collection1/data
   [junit4]   2> 1234366 T2484 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/control-001/cores/collection1/data/index/
   [junit4]   2> 1234366 T2484 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1234366 T2484 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/control-001/cores/collection1/data/index
   [junit4]   2> 1234367 T2484 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=50, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8830339632343095]
   [junit4]   2> 1234368 T2484 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@c35dcbf lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@242f85dc),segFN=segments_1,generation=1}
   [junit4]   2> 1234368 T2484 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1234373 T2484 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1234374 T2484 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1234374 T2484 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1234374 T2484 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1234374 T2484 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1234375 T2484 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1234375 T2484 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1234375 T2484 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1234376 T2484 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1234394 T2484 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1234397 T2484 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1234399 T2484 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1234401 T2484 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1234406 T2484 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update,/schema,/admin/system,/update/json/docs,/admin/mbeans,/admin/luke,/admin/ping,/admin/file,/admin/plugins,/update/csv,/get,/admin/threads,/config,/admin/logging,/admin/properties,/update/json,/admin/segments,/replication,standard
   [junit4]   2> 1234408 T2484 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1234410 T2484 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1234410 T2484 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1234412 T2484 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1234412 T2484 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1234412 T2484 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=45, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.34680924578228645]
   [junit4]   2> 1234413 T2484 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@c35dcbf lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@242f85dc),segFN=segments_1,generation=1}
   [junit4]   2> 1234413 T2484 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1234414 T2484 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@4861d637[collection1] main
   [junit4]   2> 1234414 T2484 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1234415 T2484 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1234415 T2484 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1234416 T2484 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1234416 T2484 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1234416 T2484 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1234417 T2484 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1234417 T2484 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1234417 T2484 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1234417 T2484 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1234418 T2485 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4861d637[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1234419 T2484 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1234419 T2488 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:64567 collection:control_collection shard:shard1
   [junit4]   2> 1234420 T2454 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 1234420 T2454 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1234420 T2488 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1234423 T2454 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1234424 T2454 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1234425 T2488 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1234425 T2492 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@760d8900 name:ZooKeeperConnection Watcher:127.0.0.1:12062/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1234426 T2454 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1234426 T2454 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1234426 T2454 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1234427 T2480 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1234427 T2488 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1234427 T2488 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C651 name=collection1 org.apache.solr.core.SolrCore@5cc7b0ac url=http://127.0.0.1:64567/collection1 node=127.0.0.1:64567_ C651_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:64567, node_name=127.0.0.1:64567_, core=collection1, state=down}
   [junit4]   2> 1234427 T2488 C:control_collection S:shard1 C651 P64567 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:64567/collection1/
   [junit4]   2> 1234428 T2488 C:control_collection S:shard1 C651 P64567 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1234428 T2481 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 1234428 T2488 C:control_collection S:shard1 C651 P64567 oasc.SyncStrategy.syncToMe http://127.0.0.1:64567/collection1/ has no replicas
   [junit4]   2> 1234428 T2488 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:64567/collection1/ shard1
   [junit4]   2> 1234428 T2488 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1234429 T2492 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1234429 T2480 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1234430 T2454 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1234433 T2480 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1234434 T2481 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:64567",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 1234537 T2480 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1234537 T2492 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1234586 T2488 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:64567/collection1/ and leader is http://127.0.0.1:64567/collection1/
   [junit4]   2> 1234587 T2488 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:64567
   [junit4]   2> 1234587 T2488 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1234587 T2488 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1234587 T2488 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1234588 T2480 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1234589 T2481 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:64567",
   [junit4]   2> 	  "node_name":"127.0.0.1:64567_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 3
   [junit4]   2> 1234590 T2481 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:64567",
   [junit4]   2> 	  "node_name":"127.0.0.1:64567_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"control_collection"}
   [junit4]   2> 1234693 T2480 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1234693 T2492 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1235359 T2454 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/shard-1-001/cores/collection1
   [junit4]   2> 1235361 T2454 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/shard-1-001
   [junit4]   2> 1235361 T2454 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1235364 T2454 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:11584
   [junit4]   2> 1235364 T2454 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, hostPort=28025, solrconfig=solrconfig.xml, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/shard-1-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/tempDir-001/jetty1}
   [junit4]   2> 1235365 T2454 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 1235365 T2454 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/shard-1-001/'
   [junit4]   2> 1235394 T2454 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/shard-1-001/solr.xml
   [junit4]   2> 1235428 T2454 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/shard-1-001/cores
   [junit4]   2> 1235429 T2454 oasc.CoreContainer.<init> New CoreContainer 1578453578
   [junit4]   2> 1235429 T2454 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/shard-1-001/]
   [junit4]   2> 1235429 T2454 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/shard-1-001/lib
   [junit4]   2> 1235430 T2454 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/shard-1-001/lib).
   [junit4]   2> 1235438 T2454 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1235439 T2454 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1235439 T2454 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1235439 T2454 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1235439 T2454 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1235439 T2454 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1235440 T2454 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1235440 T2454 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1235440 T2454 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1235440 T2454 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1235441 T2454 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1235441 T2454 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1235441 T2454 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1235441 T2454 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1235442 T2454 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1235442 T2454 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:12062/solr
   [junit4]   2> 1235442 T2454 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1235442 T2454 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1235443 T2454 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1235445 T2504 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5417ca42 name:ZooKeeperConnection Watcher:127.0.0.1:12062 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1235446 T2454 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1235447 T2454 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1235449 T2454 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1235450 T2507 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@58728c97 name:ZooKeeperConnection Watcher:127.0.0.1:12062/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1235450 T2454 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1235455 T2454 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1236460 T2454 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:28025_
   [junit4]   2> 1236461 T2454 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:28025_
   [junit4]   2> 1236464 T2454 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1236466 T2454 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/shard-1-001/cores
   [junit4]   2> 1236468 T2454 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, dataDir=data/, coreNodeName=, config=solrconfig.xml, collection=collection1, schema=schema.xml, shard=, loadOnStartup=true, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/shard-1-001/cores/collection1/, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/shard-1-001/cores/collection1}
   [junit4]   2> 1236468 T2454 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/shard-1-001/cores/collection1/
   [junit4]   2> 1236468 T2454 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1236469 T2508 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1236469 T2508 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1236470 T2480 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1236470 T2508 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1236471 T2481 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28025",
   [junit4]   2> 	  "node_name":"127.0.0.1:28025_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1"} current state version: 4
   [junit4]   2> 1236472 T2481 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28025",
   [junit4]   2> 	  "node_name":"127.0.0.1:28025_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1"}
   [junit4]   2> 1236472 T2481 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 1236472 T2481 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1236473 T2492 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1236474 T2507 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1236473 T2480 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1237471 T2508 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1237472 T2508 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1237473 T2508 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1237473 T2508 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1237473 T2508 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1237474 T2508 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/shard-1-001/cores/collection1/'
   [junit4]   2> 1237486 T2508 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1237488 T2508 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1237496 T2508 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1237510 T2508 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 1237543 T2508 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1237645 T2508 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1237652 T2508 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1237919 T2508 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1237930 T2508 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1237932 T2508 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1237941 T2508 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1237944 T2508 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1237947 T2508 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1237947 T2508 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1237948 T2508 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1237948 T2508 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1237948 T2508 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1237949 T2508 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1237949 T2508 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1237949 T2508 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1237949 T2508 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 1237949 T2508 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@21018702
   [junit4]   2> 1237950 T2508 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/shard-1-001/cores/collection1/data
   [junit4]   2> 1237950 T2508 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 1237950 T2508 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1237951 T2508 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 2EFCD7E4519914F5-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 1237951 T2508 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=50, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8830339632343095]
   [junit4]   2> 1237952 T2508 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@48137c2 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3f73a3dd),segFN=segments_1,generation=1}
   [junit4]   2> 1237952 T2508 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1237956 T2508 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1237957 T2508 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1237957 T2508 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1237957 T2508 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1237957 T2508 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1237958 T2508 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1237958 T2508 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1237958 T2508 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1237958 T2508 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1237970 T2508 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1237972 T2508 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1237973 T2508 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1237975 T2508 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1237980 T2508 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update,/schema,/admin/system,/update/json/docs,/admin/mbeans,/admin/luke,/admin/ping,/admin/file,/admin/plugins,/update/csv,/get,/admin/threads,/config,/admin/logging,/admin/properties,/update/json,/admin/segments,/replication,standard
   [junit4]   2> 1237981 T2508 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1237983 T2508 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1237983 T2508 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1237984 T2508 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1237984 T2508 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1237985 T2508 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=45, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.34680924578228645]
   [junit4]   2> 1237986 T2508 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@48137c2 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3f73a3dd),segFN=segments_1,generation=1}
   [junit4]   2> 1237986 T2508 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1237986 T2508 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@79da4980[collection1] main
   [junit4]   2> 1237986 T2508 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1237987 T2508 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1237987 T2508 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1237988 T2508 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1237988 T2508 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1237988 T2508 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1237989 T2508 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1237989 T2508 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1237989 T2508 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1237989 T2508 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1237990 T2509 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@79da4980[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1237991 T2508 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1237991 T2512 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:28025 collection:collection1 shard:shard1
   [junit4]   2> 1237992 T2454 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 1237992 T2454 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1237992 T2512 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1237997 T2512 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1237998 T2480 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1237999 T2512 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1237999 T2512 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C652 name=collection1 org.apache.solr.core.SolrCore@5fa6cb4d url=http://127.0.0.1:28025/collection1 node=127.0.0.1:28025_ C652_STATE=coll:collection1 core:collection1 props:{base_url=http://127.0.0.1:28025, node_name=127.0.0.1:28025_, core=collection1, state=down}
   [junit4]   2> 1237999 T2512 C:collection1 S:shard1 C652 P28025 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:28025/collection1/
   [junit4]   2> 1237999 T2481 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 1237999 T2512 C:collection1 S:shard1 C652 P28025 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1238000 T2512 C:collection1 S:shard1 C652 P28025 oasc.SyncStrategy.syncToMe http://127.0.0.1:28025/collection1/ has no replicas
   [junit4]   2> 1238000 T2512 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:28025/collection1/ shard1
   [junit4]   2> 1238000 T2512 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1238001 T2480 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1238001 T2492 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1238001 T2507 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1238005 T2480 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1238005 T2481 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28025",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 1238108 T2480 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1238108 T2507 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1238108 T2492 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1238157 T2512 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:28025/collection1/ and leader is http://127.0.0.1:28025/collection1/
   [junit4]   2> 1238158 T2512 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:28025
   [junit4]   2> 1238158 T2512 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1238158 T2512 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 1238158 T2512 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1238159 T2480 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1238160 T2481 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28025",
   [junit4]   2> 	  "node_name":"127.0.0.1:28025_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 7
   [junit4]   2> 1238160 T2481 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28025",
   [junit4]   2> 	  "node_name":"127.0.0.1:28025_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "co

[...truncated too long message...]

: http://127.0.0.1:48728/
   [junit4]   2> 1554754 T4857 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 1 connections to: http://127.0.0.1:48718/
   [junit4]   2> 1554754 T4857 C:control_collection S:shard1 c:collection1 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 1554755 T4857 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:48738/
   [junit4]   2> 1554755 T4857 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:48754/
   [junit4]   2> 1554755 T4857 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:48728/
   [junit4]   2> 1554755 T4857 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:48718/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=2EFCD7E4519914F5 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=be -Dtests.timezone=Pacific/Guadalcanal -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   79.7s J1 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48738/repfacttest_c8n_1x3_shard1_replica3
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([2EFCD7E4519914F5:A6A8E83EFF65790D]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48738/repfacttest_c8n_1x3_shard1_replica3
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 48 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 53 more
   [junit4]   2> 1554831 T4857 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 2EFCD7E4519914F5-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {id=PostingsFormat(name=Asserting), intDefault=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), timestamp=PostingsFormat(name=LuceneVarGapFixedInterval), a_t=PostingsFormat(name=LuceneVarGapFixedInterval), multiDefault=PostingsFormat(name=LuceneVarGapFixedInterval), text=PostingsFormat(name=Memory doPackFST= true), range_facet_l=PostingsFormat(name=LuceneVarGapFixedInterval), _version_=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128)))}, docValues:{timestamp=DocValuesFormat(name=Lucene50)}, sim=RandomSimilarityProvider(queryNorm=false,coord=crazy): {}, locale=be, timezone=Pacific/Guadalcanal
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=183992848,total=381157376
   [junit4]   2> NOTE: All tests run in this JVM: [FileBasedSpellCheckerTest, TestExceedMaxTermLength, SimpleMLTQParserTest, RollingRestartTest, TestSolrConfigHandlerCloud, TestSolrJ, TestRealTimeGet, SolrPluginUtilsTest, ResourceLoaderTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, DateRangeFieldTest, TestUniqueKeyFieldResource, TestUtils, TestOmitPositions, TestFoldingMultitermQuery, SolrCmdDistributorTest, TestFastWriter, HardAutoCommitTest, TestSchemaSimilarityResource, TestIBSimilarityFactory, MoreLikeThisHandlerTest, SortByFunctionTest, TestReRankQParserPlugin, CollectionsAPIDistributedZkTest, ChaosMonkeyNothingIsSafeTest, SolrCoreCheckLockOnStartupTest, MultiTermTest, SegmentsInfoRequestHandlerTest, SolrRequestParserTest, TestJmxIntegration, SimpleFacetsTest, BadComponentTest, PathHierarchyTokenizerFactoryTest, TestDistributedGrouping, DocExpirationUpdateProcessorFactoryTest, ShardRoutingCustomTest, TestQuerySenderNoQuery, TestRequestStatusCollectionAPI, DistributedFacetPivotWhiteBoxTest, CoreAdminRequestStatusTest, TestReload, TestFiltering, SolrInfoMBeanTest, TestCollapseQParserPlugin, AsyncMigrateRouteKeyTest, TestCSVResponseWriter, TestStressUserVersions, DeleteInactiveReplicaTest, TestObjectReleaseTracker, CloudExitableDirectoryReaderTest, TermVectorComponentTest, TestFastLRUCache, TestJsonRequest, TestHashPartitioner, TermVectorComponentDistributedTest, TestExactStatsCache, BasicDistributedZk2Test, TestReplicationHandler, BasicDistributedZkTest, ReplicationFactorTest]
   [junit4] Completed on J1 in 79.71s, 1 test, 1 error <<< FAILURES!

[...truncated 714 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 62 ignored (22 assumptions)

Total time: 54 minutes 43 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 218884278 bytes
Compression is 0.0%
Took 40 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2854 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2854/

4 tests failed.
REGRESSION:  org.apache.solr.cloud.TestReplicaProperties.test

Error Message:
Collection testcollection does not have roles evenly distributed. Collection is: DocCollection(testcollection)={   "shards":{     "shard1":{       "range":"80000000-b332ffff",       "state":"active",       "replicas":{         "core_node1":{           "core":"testcollection_shard1_replica1",           "base_url":"http://127.0.0.1:20397/v/uy",           "state":"active",           "node_name":"127.0.0.1:20397_v%2Fuy",           "leader":"true"},         "core_node3":{           "core":"testcollection_shard1_replica3",           "base_url":"http://127.0.0.1:20421/v/uy",           "state":"active",           "node_name":"127.0.0.1:20421_v%2Fuy",           "property.preferredleader":"true"},         "core_node4":{           "core":"testcollection_shard1_replica2",           "base_url":"http://127.0.0.1:20438/v/uy",           "state":"active",           "node_name":"127.0.0.1:20438_v%2Fuy"}}},     "shard2":{       "range":"b3330000-e665ffff",       "state":"active",       "replicas":{         "core_node2":{           "core":"testcollection_shard2_replica1",           "base_url":"http://127.0.0.1:20403/v/uy",           "state":"active",           "node_name":"127.0.0.1:20403_v%2Fuy"},         "core_node5":{           "core":"testcollection_shard2_replica2",           "base_url":"http://127.0.0.1:20464/v/uy",           "state":"active",           "node_name":"127.0.0.1:20464_v%2Fuy",           "property.preferredleader":"true"},         "core_node7":{           "core":"testcollection_shard2_replica3",           "base_url":"http://127.0.0.1:20397/v/uy",           "state":"active",           "node_name":"127.0.0.1:20397_v%2Fuy",           "leader":"true"}}},     "shard3":{       "range":"e6660000-1998ffff",       "state":"active",       "replicas":{         "core_node6":{           "core":"testcollection_shard3_replica1",           "base_url":"http://127.0.0.1:20438/v/uy",           "state":"active",           "node_name":"127.0.0.1:20438_v%2Fuy"},         "core_node9":{           "core":"testcollection_shard3_replica2",           "base_url":"http://127.0.0.1:20421/v/uy",           "state":"active",           "node_name":"127.0.0.1:20421_v%2Fuy",           "leader":"true"},         "core_node13":{           "core":"testcollection_shard3_replica3",           "base_url":"http://127.0.0.1:20403/v/uy",           "state":"active",           "node_name":"127.0.0.1:20403_v%2Fuy",           "property.preferredleader":"true"}}},     "shard4":{       "range":"19990000-4ccbffff",       "state":"active",       "replicas":{         "core_node8":{           "core":"testcollection_shard4_replica1",           "base_url":"http://127.0.0.1:20464/v/uy",           "state":"active",           "node_name":"127.0.0.1:20464_v%2Fuy"},         "core_node10":{           "core":"testcollection_shard4_replica2",           "base_url":"http://127.0.0.1:20397/v/uy",           "state":"active",           "node_name":"127.0.0.1:20397_v%2Fuy",           "leader":"true"},         "core_node12":{           "core":"testcollection_shard4_replica3",           "base_url":"http://127.0.0.1:20438/v/uy",           "state":"active",           "node_name":"127.0.0.1:20438_v%2Fuy",           "property.preferredleader":"true"}}},     "shard5":{       "range":"4ccc0000-7fffffff",       "state":"active",       "replicas":{         "core_node11":{           "core":"testcollection_shard5_replica1",           "base_url":"http://127.0.0.1:20421/v/uy",           "state":"active",           "node_name":"127.0.0.1:20421_v%2Fuy"},         "core_node14":{           "core":"testcollection_shard5_replica2",           "base_url":"http://127.0.0.1:20403/v/uy",           "state":"active",           "node_name":"127.0.0.1:20403_v%2Fuy",           "leader":"true"},         "core_node15":{           "core":"testcollection_shard5_replica3",           "base_url":"http://127.0.0.1:20464/v/uy",           "state":"active",           "node_name":"127.0.0.1:20464_v%2Fuy",           "property.preferredleader":"true"}}}},   "replicationFactor":"3",   "autoAddReplicas":"false",   "maxShardsPerNode":"16",   "router":{"name":"compositeId"}}

Stack Trace:
java.lang.AssertionError: Collection testcollection does not have roles evenly distributed. Collection is: DocCollection(testcollection)={
  "shards":{
    "shard1":{
      "range":"80000000-b332ffff",
      "state":"active",
      "replicas":{
        "core_node1":{
          "core":"testcollection_shard1_replica1",
          "base_url":"http://127.0.0.1:20397/v/uy",
          "state":"active",
          "node_name":"127.0.0.1:20397_v%2Fuy",
          "leader":"true"},
        "core_node3":{
          "core":"testcollection_shard1_replica3",
          "base_url":"http://127.0.0.1:20421/v/uy",
          "state":"active",
          "node_name":"127.0.0.1:20421_v%2Fuy",
          "property.preferredleader":"true"},
        "core_node4":{
          "core":"testcollection_shard1_replica2",
          "base_url":"http://127.0.0.1:20438/v/uy",
          "state":"active",
          "node_name":"127.0.0.1:20438_v%2Fuy"}}},
    "shard2":{
      "range":"b3330000-e665ffff",
      "state":"active",
      "replicas":{
        "core_node2":{
          "core":"testcollection_shard2_replica1",
          "base_url":"http://127.0.0.1:20403/v/uy",
          "state":"active",
          "node_name":"127.0.0.1:20403_v%2Fuy"},
        "core_node5":{
          "core":"testcollection_shard2_replica2",
          "base_url":"http://127.0.0.1:20464/v/uy",
          "state":"active",
          "node_name":"127.0.0.1:20464_v%2Fuy",
          "property.preferredleader":"true"},
        "core_node7":{
          "core":"testcollection_shard2_replica3",
          "base_url":"http://127.0.0.1:20397/v/uy",
          "state":"active",
          "node_name":"127.0.0.1:20397_v%2Fuy",
          "leader":"true"}}},
    "shard3":{
      "range":"e6660000-1998ffff",
      "state":"active",
      "replicas":{
        "core_node6":{
          "core":"testcollection_shard3_replica1",
          "base_url":"http://127.0.0.1:20438/v/uy",
          "state":"active",
          "node_name":"127.0.0.1:20438_v%2Fuy"},
        "core_node9":{
          "core":"testcollection_shard3_replica2",
          "base_url":"http://127.0.0.1:20421/v/uy",
          "state":"active",
          "node_name":"127.0.0.1:20421_v%2Fuy",
          "leader":"true"},
        "core_node13":{
          "core":"testcollection_shard3_replica3",
          "base_url":"http://127.0.0.1:20403/v/uy",
          "state":"active",
          "node_name":"127.0.0.1:20403_v%2Fuy",
          "property.preferredleader":"true"}}},
    "shard4":{
      "range":"19990000-4ccbffff",
      "state":"active",
      "replicas":{
        "core_node8":{
          "core":"testcollection_shard4_replica1",
          "base_url":"http://127.0.0.1:20464/v/uy",
          "state":"active",
          "node_name":"127.0.0.1:20464_v%2Fuy"},
        "core_node10":{
          "core":"testcollection_shard4_replica2",
          "base_url":"http://127.0.0.1:20397/v/uy",
          "state":"active",
          "node_name":"127.0.0.1:20397_v%2Fuy",
          "leader":"true"},
        "core_node12":{
          "core":"testcollection_shard4_replica3",
          "base_url":"http://127.0.0.1:20438/v/uy",
          "state":"active",
          "node_name":"127.0.0.1:20438_v%2Fuy",
          "property.preferredleader":"true"}}},
    "shard5":{
      "range":"4ccc0000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node11":{
          "core":"testcollection_shard5_replica1",
          "base_url":"http://127.0.0.1:20421/v/uy",
          "state":"active",
          "node_name":"127.0.0.1:20421_v%2Fuy"},
        "core_node14":{
          "core":"testcollection_shard5_replica2",
          "base_url":"http://127.0.0.1:20403/v/uy",
          "state":"active",
          "node_name":"127.0.0.1:20403_v%2Fuy",
          "leader":"true"},
        "core_node15":{
          "core":"testcollection_shard5_replica3",
          "base_url":"http://127.0.0.1:20464/v/uy",
          "state":"active",
          "node_name":"127.0.0.1:20464_v%2Fuy",
          "property.preferredleader":"true"}}}},
  "replicationFactor":"3",
  "autoAddReplicas":"false",
  "maxShardsPerNode":"16",
  "router":{"name":"compositeId"}}
	at __randomizedtesting.SeedInfo.seed([50DDE0681DAB4866:D889DFB2B357259E]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.ReplicaPropertiesBase.verifyUnique(ReplicaPropertiesBase.java:176)
	at org.apache.solr.cloud.ReplicaPropertiesBase.verifyUniqueAcrossCollection(ReplicaPropertiesBase.java:107)
	at org.apache.solr.cloud.TestReplicaProperties.clusterAssignPropertyTest(TestReplicaProperties.java:108)
	at org.apache.solr.cloud.TestReplicaProperties.test(TestReplicaProperties.java:70)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:49343/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:49343/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([50DDE0681DAB4866:D889DFB2B357259E]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([50DDE0681DAB4866:D889DFB2B357259E]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([50DDE0681DAB4866:D889DFB2B357259E]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9271 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 50DDE0681DAB4866-001/init-core-data-001
   [junit4]   2> 100981 T235 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 101012 T235 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 101019 T236 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 101219 T235 oasc.ZkTestServer.run start zk server on port:49319
   [junit4]   2> 101345 T235 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 101570 T235 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 101723 T243 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6746bb84 name:ZooKeeperConnection Watcher:127.0.0.1:49319 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 101724 T235 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 101725 T235 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 101736 T235 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 101853 T235 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 101854 T235 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 101855 T246 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2075445e name:ZooKeeperConnection Watcher:127.0.0.1:49319/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 101856 T235 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 101857 T235 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 101864 T235 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 101868 T235 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 101871 T235 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 101874 T235 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 101878 T235 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 101878 T235 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 101884 T235 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 101884 T235 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 101888 T235 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 101889 T235 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 101893 T235 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 101893 T235 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 101897 T235 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 101897 T235 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 101901 T235 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 101901 T235 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 101905 T235 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 101905 T235 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 101909 T235 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 101909 T235 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 101913 T235 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 101913 T235 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 101917 T235 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 101917 T235 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 101921 T235 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 101921 T235 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 103263 T235 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 50DDE0681DAB4866-001/control-001/cores/collection1
   [junit4]   2> 103267 T235 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 103271 T235 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:49328
   [junit4]   2> 103271 T235 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=49327, hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 50DDE0681DAB4866-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 50DDE0681DAB4866-001/tempDir-001/control/data}
   [junit4]   2> 103272 T235 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 103272 T235 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 50DDE0681DAB4866-001/control-001/'
   [junit4]   2> 103302 T235 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 50DDE0681DAB4866-001/control-001/solr.xml
   [junit4]   2> 103338 T235 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 50DDE0681DAB4866-001/control-001/cores
   [junit4]   2> 103338 T235 oasc.CoreContainer.<init> New CoreContainer 567045674
   [junit4]   2> 103339 T235 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 50DDE0681DAB4866-001/control-001/]
   [junit4]   2> 103339 T235 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 50DDE0681DAB4866-001/control-001/lib
   [junit4]   2> 103340 T235 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 50DDE0681DAB4866-001/control-001/lib).
   [junit4]   2> 103349 T235 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 103349 T235 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 103349 T235 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 103350 T235 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 103350 T235 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 103350 T235 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 103351 T235 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 103351 T235 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 103351 T235 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 103351 T235 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 103352 T235 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 103352 T235 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 103353 T235 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 103353 T235 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 103353 T235 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 103354 T235 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49319/solr
   [junit4]   2> 103389 T235 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 103389 T235 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 103390 T235 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 103392 T258 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@64ee13f name:ZooKeeperConnection Watcher:127.0.0.1:49319 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 103392 T235 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 103393 T235 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 103405 T235 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 103407 T261 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6d886777 name:ZooKeeperConnection Watcher:127.0.0.1:49319/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 103407 T235 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 103421 T235 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 103432 T235 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 103440 T235 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 103443 T235 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 103446 T235 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 103465 T235 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 103468 T235 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 103470 T235 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 103472 T235 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49327_
   [junit4]   2> 103478 T235 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49327_
   [junit4]   2> 103481 T235 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 103483 T235 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 103485 T235 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 103500 T235 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:49327_
   [junit4]   2> 103500 T235 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 103503 T235 oasc.Overseer.start Overseer (id=93562377799335939-127.0.0.1:49327_-n_0000000000) starting
   [junit4]   2> 103517 T235 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 103543 T235 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 103557 T263 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 103557 T235 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 103575 T262 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 103580 T235 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 50DDE0681DAB4866-001/control-001/cores
   [junit4]   2> 103591 T235 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {coreNodeName=, config=solrconfig.xml, schema=schema.xml, collection=control_collection, name=collection1, loadOnStartup=true, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 50DDE0681DAB4866-001/control-001/cores/collection1/, shard=, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 50DDE0681DAB4866-001/control-001/cores/collection1, dataDir=data/}
   [junit4]   2> 103591 T235 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 50DDE0681DAB4866-001/control-001/cores/collection1/
   [junit4]   2> 103592 T235 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 103593 T265 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 103593 T265 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 103604 T261 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 103604 T265 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 103609 T262 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49327",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:49327_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down"} current state version: 0
   [junit4]   2> 103643 T262 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49327",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:49327_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 103648 T262 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 103677 T262 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 103688 T261 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 104607 T265 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 104607 T265 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 104608 T265 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 104609 T265 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 104610 T265 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 104611 T265 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 50DDE0681DAB4866-001/control-001/cores/collection1/'
   [junit4]   2> 104671 T265 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 104702 T265 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 104744 T265 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 104801 T265 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 104863 T265 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 104866 T265 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 104880 T265 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 105556 T265 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 105576 T265 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 105579 T265 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 105605 T265 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 105611 T265 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 105616 T265 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 105617 T265 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 105618 T265 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 105618 T265 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 105620 T265 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 105620 T265 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 105620 T265 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 105621 T265 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 105622 T265 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 50DDE0681DAB4866-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 50DDE0681DAB4866-001/control-001/cores/collection1/data/
   [junit4]   2> 105622 T265 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5fdb47b9
   [junit4]   2> 105637 T265 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 50DDE0681DAB4866-001/control-001/cores/collection1/data
   [junit4]   2> 105638 T265 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 50DDE0681DAB4866-001/control-001/cores/collection1/data/index/
   [junit4]   2> 105638 T265 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 50DDE0681DAB4866-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 105639 T265 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 50DDE0681DAB4866-001/control-001/cores/collection1/data/index
   [junit4]   2> 105639 T265 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=31, maxMergedSegmentMB=60.37109375, floorSegmentMB=0.2880859375, forceMergeDeletesPctAllowed=26.270123372721947, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5420063929648241
   [junit4]   2> 105640 T265 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1548d6e2 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@61139a58),segFN=segments_1,generation=1}
   [junit4]   2> 105641 T265 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 105653 T265 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 105724 T265 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 105724 T265 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 105725 T265 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 105725 T265 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 105726 T265 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 105726 T265 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 105727 T265 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 105727 T265 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 105748 T265 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 105763 T265 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 105767 T265 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 105771 T265 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 105789 T265 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/properties,/update,/admin/file,/admin/system,/admin/ping,/admin/mbeans,/schema,/admin/logging,/admin/threads,/config,/get,/admin/luke,/update/json/docs,standard,/update/csv,/update/json,/replication,/admin/plugins,/admin/segments
   [junit4]   2> 105792 T265 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 105793 T265 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 105794 T265 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 105795 T265 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 105796 T265 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 105798 T265 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=25, maxMergedSegmentMB=74.40234375, floorSegmentMB=1.041015625, forceMergeDeletesPctAllowed=5.290010182090534, segmentsPerTier=16.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 105799 T265 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1548d6e2 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@61139a58),segFN=segments_1,generation=1}
   [junit4]   2> 105799 T265 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 105800 T265 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@2e3b3e5d[collection1] main
   [junit4]   2> 105800 T265 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 105802 T265 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 105802 T265 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 105824 T265 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 105824 T265 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 105825 T265 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 105826 T265 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 105827 T265 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 105827 T265 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 105827 T265 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 105830 T266 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2e3b3e5d[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 105837 T265 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 105838 T269 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:49327 collection:control_collection shard:shard1
   [junit4]   2> 105839 T235 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 105840 T235 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 105868 T269 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 105877 T269 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 105880 T261 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 105881 T269 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 105881 T269 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 105882 T262 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C22 name=collection1 org.apache.solr.core.SolrCore@6780b40d url=http://127.0.0.1:49327/collection1 node=127.0.0.1:49327_ C22_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:49327, node_name=127.0.0.1:49327_, core=collection1, state=down}
   [junit4]   2> 105882 T269 C:control_collection S:shard1 C22 P49327 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:49327/collection1/
   [junit4]   2> 105883 T269 C:control_collection S:shard1 C22 P49327 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 105883 T269 C:control_collection S:shard1 C22 P49327 oasc.SyncStrategy.syncToMe http://127.0.0.1:49327/collection1/ has no replicas
   [junit4]   2> 105883 T269 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:49327/collection1/ shard1
   [junit4]   2> 105888 T235 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 105889 T235 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 105891 T273 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2247a195 name:ZooKeeperConnection Watcher:127.0.0.1:49319/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 105892 T235 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 105892 T235 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 105893 T235 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 105893 T261 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 105897 T269 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 105902 T235 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 105903 T235 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 105903 T235 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 105904 T235 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 105905 T261 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 105906 T276 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4d7dac8a name:ZooKeeperConnection Watcher:127.0.0.1:49319/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 105906 T235 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 105907 T235 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 105907 T262 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49327",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 105913 T262 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 105913 T262 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 105914 T261 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 105914 T273 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 105921 T261 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 105923 T262 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 105925 T273 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 105925 T261 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 105957 T269 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:49327/collection1/ and leader is http://127.0.0.1:49327/collection1/
   [junit4]   2> 105958 T269 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:49327
   [junit4]   2> 105958 T269 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 105958 T269 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 105958 T269 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 105960 T261 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 105962 T262 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49327",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:49327_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active"} current state version: 4
   [junit4]   2> 105963 T262 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49327",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:49327_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active"}
   [junit4]   2> 106069 T261 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 106069 T273 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 107158 T235 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 50DDE0681DAB4866-001/shard-1-001/cores/collection1
   [junit4]   2> 107161 T235 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 50DDE0681DAB4866-001/shard-1-001
   [junit4]   2> 107162 T235 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 107164 T235 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:49344
   [junit4]   2> 107164 T235 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 50DDE0681DAB4866-001/tempDir-001/jetty1, hostContext=/, hostPort=49343, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 50DDE0681DAB4866-001/shard-1-001/cores, solrconfig=solrconfig.xml}
   [junit4]   2> 107164 T235 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 107165 T235 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 50DDE0681DAB4866-001/shard-1-001/'
   [junit4]   2> 107194 T235 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 50DDE0681DAB4866-001/shard-1-001/solr.xml
   [junit4]   2> 107231 T235 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 50DDE0681DAB4866-001/shard-1-001/cores
   [junit4]   2> 107231 T235 oasc.CoreContainer.<init> New CoreContainer 769549428
   [junit4]   2> 107232 T235 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 50DDE0681DAB4866-001/shard-1-001/]
   [junit4]   2> 107232 T235 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 50DDE0681DAB4866-001/shard-1-001/lib
   [junit4]   2> 107233 T235 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 50DDE0681DAB4866-001/shard-1-001/lib).
   [junit4]   2> 107242 T235 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 107242 T235 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 107243 T235 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 107243 T235 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 107243 T235 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 107243 T235 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 107244 T235 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 107244 T235 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 107244 T235 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 107244 T235 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 107245 T235 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 107245 T235 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 107246 T235 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 107246 T235 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 107246 T235 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 107246 T235 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49319/solr
   [junit4]   2> 107247 T235 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 107247 T235 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 107248 T235 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 107250 T288 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5f118604 name:ZooKeeperConnection Watcher:127.0.0.1:49319 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 107250 T235 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 107251 T235 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 107253 T235 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 107254 T291 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2fdc4517 name:ZooKeeperConnection Watcher:127.0.0.1:49319/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 107255 T235 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 107261 T235 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 108271 T235 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49343_
   [junit4]   2> 108272 T235 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49343_
   [junit4]   2> 108277 T235 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 108290 T235 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 50DDE0681DAB4866-001/shard-1-001/cores
   [junit4]   2> 108293 T235 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=collection1, name=collection1, shard=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 50DDE0681DAB4866-001/shard-1-001/cores/collection1/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 50DDE0681DAB4866-001/shard-1-001/cores/collection1, dataDir=data/, schema=schema.xml, loadOnStartup=true, config=solrconfig.xml, transient=false, coreNodeName=}
   [junit4]   2> 108294 T235 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 50DDE0681DAB4866-001/shard-1-001/cores/collection1/
   [junit4]   2> 108294 T235 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 108299 T292 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 108299 T292 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 108302 T261 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 108304 T292 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 108304 T262 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49343",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:49343_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down"} current state version: 5
   [junit4]   2> 108304 T292 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 108305 T262 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49343",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:49343_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 108305 T292 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 108306 T262 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 108306 T262 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 108310 T262 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 108315 T292 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 108315 T292 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 108315 T292 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 108316 T292 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 108317 T292 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 108317 T292 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 108319 T292 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 108319 T292 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 50DDE0681DAB4866-001/shard-1-001/cores/collection1/'
   [junit4]   2> 108337 T292 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 108341 T292 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 108352 T292 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 108372 T292 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 108409 T292 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 108412 T292 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 108421 T292 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 108738 T292 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 108752 T292 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 108755 T292 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 108777 T292 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 108782 T292 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 108787 T292 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 108788 T292 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 108789 T292 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 108789 T292 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 108791 T292 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 108791 T292 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 108791 T292 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 108792 T292 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 108792 T292 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 50DDE0681DAB4866-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 50DDE0681DAB4866-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 108792 T292 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5fdb47b9
   [junit4]   2> 108793 T292 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 50DDE0681DAB4866-001/shard-1-001/cores/collection1/data
   [junit4]   2> 108793 T292 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1

[...truncated too long message...]

           "core":"testcollection_shard1_replica2",
   [junit4]    >           "base_url":"http://127.0.0.1:20438/v/uy",
   [junit4]    >           "state":"active",
   [junit4]    >           "node_name":"127.0.0.1:20438_v%2Fuy"}}},
   [junit4]    >     "shard2":{
   [junit4]    >       "range":"b3330000-e665ffff",
   [junit4]    >       "state":"active",
   [junit4]    >       "replicas":{
   [junit4]    >         "core_node2":{
   [junit4]    >           "core":"testcollection_shard2_replica1",
   [junit4]    >           "base_url":"http://127.0.0.1:20403/v/uy",
   [junit4]    >           "state":"active",
   [junit4]    >           "node_name":"127.0.0.1:20403_v%2Fuy"},
   [junit4]    >         "core_node5":{
   [junit4]    >           "core":"testcollection_shard2_replica2",
   [junit4]    >           "base_url":"http://127.0.0.1:20464/v/uy",
   [junit4]    >           "state":"active",
   [junit4]    >           "node_name":"127.0.0.1:20464_v%2Fuy",
   [junit4]    >           "property.preferredleader":"true"},
   [junit4]    >         "core_node7":{
   [junit4]    >           "core":"testcollection_shard2_replica3",
   [junit4]    >           "base_url":"http://127.0.0.1:20397/v/uy",
   [junit4]    >           "state":"active",
   [junit4]    >           "node_name":"127.0.0.1:20397_v%2Fuy",
   [junit4]    >           "leader":"true"}}},
   [junit4]    >     "shard3":{
   [junit4]    >       "range":"e6660000-1998ffff",
   [junit4]    >       "state":"active",
   [junit4]    >       "replicas":{
   [junit4]    >         "core_node6":{
   [junit4]    >           "core":"testcollection_shard3_replica1",
   [junit4]    >           "base_url":"http://127.0.0.1:20438/v/uy",
   [junit4]    >           "state":"active",
   [junit4]    >           "node_name":"127.0.0.1:20438_v%2Fuy"},
   [junit4]    >         "core_node9":{
   [junit4]    >           "core":"testcollection_shard3_replica2",
   [junit4]    >           "base_url":"http://127.0.0.1:20421/v/uy",
   [junit4]    >           "state":"active",
   [junit4]    >           "node_name":"127.0.0.1:20421_v%2Fuy",
   [junit4]    >           "leader":"true"},
   [junit4]    >         "core_node13":{
   [junit4]    >           "core":"testcollection_shard3_replica3",
   [junit4]    >           "base_url":"http://127.0.0.1:20403/v/uy",
   [junit4]    >           "state":"active",
   [junit4]    >           "node_name":"127.0.0.1:20403_v%2Fuy",
   [junit4]    >           "property.preferredleader":"true"}}},
   [junit4]    >     "shard4":{
   [junit4]    >       "range":"19990000-4ccbffff",
   [junit4]    >       "state":"active",
   [junit4]    >       "replicas":{
   [junit4]    >         "core_node8":{
   [junit4]    >           "core":"testcollection_shard4_replica1",
   [junit4]    >           "base_url":"http://127.0.0.1:20464/v/uy",
   [junit4]    >           "state":"active",
   [junit4]    >           "node_name":"127.0.0.1:20464_v%2Fuy"},
   [junit4]    >         "core_node10":{
   [junit4]    >           "core":"testcollection_shard4_replica2",
   [junit4]    >           "base_url":"http://127.0.0.1:20397/v/uy",
   [junit4]    >           "state":"active",
   [junit4]    >           "node_name":"127.0.0.1:20397_v%2Fuy",
   [junit4]    >           "leader":"true"},
   [junit4]    >         "core_node12":{
   [junit4]    >           "core":"testcollection_shard4_replica3",
   [junit4]    >           "base_url":"http://127.0.0.1:20438/v/uy",
   [junit4]    >           "state":"active",
   [junit4]    >           "node_name":"127.0.0.1:20438_v%2Fuy",
   [junit4]    >           "property.preferredleader":"true"}}},
   [junit4]    >     "shard5":{
   [junit4]    >       "range":"4ccc0000-7fffffff",
   [junit4]    >       "state":"active",
   [junit4]    >       "replicas":{
   [junit4]    >         "core_node11":{
   [junit4]    >           "core":"testcollection_shard5_replica1",
   [junit4]    >           "base_url":"http://127.0.0.1:20421/v/uy",
   [junit4]    >           "state":"active",
   [junit4]    >           "node_name":"127.0.0.1:20421_v%2Fuy"},
   [junit4]    >         "core_node14":{
   [junit4]    >           "core":"testcollection_shard5_replica2",
   [junit4]    >           "base_url":"http://127.0.0.1:20403/v/uy",
   [junit4]    >           "state":"active",
   [junit4]    >           "node_name":"127.0.0.1:20403_v%2Fuy",
   [junit4]    >           "leader":"true"},
   [junit4]    >         "core_node15":{
   [junit4]    >           "core":"testcollection_shard5_replica3",
   [junit4]    >           "base_url":"http://127.0.0.1:20464/v/uy",
   [junit4]    >           "state":"active",
   [junit4]    >           "node_name":"127.0.0.1:20464_v%2Fuy",
   [junit4]    >           "property.preferredleader":"true"}}}},
   [junit4]    >   "replicationFactor":"3",
   [junit4]    >   "autoAddReplicas":"false",
   [junit4]    >   "maxShardsPerNode":"16",
   [junit4]    >   "router":{"name":"compositeId"}}
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([50DDE0681DAB4866:D889DFB2B357259E]:0)
   [junit4]    > 	at org.apache.solr.cloud.ReplicaPropertiesBase.verifyUnique(ReplicaPropertiesBase.java:176)
   [junit4]    > 	at org.apache.solr.cloud.ReplicaPropertiesBase.verifyUniqueAcrossCollection(ReplicaPropertiesBase.java:107)
   [junit4]    > 	at org.apache.solr.cloud.TestReplicaProperties.clusterAssignPropertyTest(TestReplicaProperties.java:108)
   [junit4]    > 	at org.apache.solr.cloud.TestReplicaProperties.test(TestReplicaProperties.java:70)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1771425 T3545 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.TestReplicaProperties 50DDE0681DAB4866-001
   [junit4]   2> 70497 T3544 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=false,coord=crazy): {}, locale=zh_HK, timezone=America/Juneau
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=81022888,total=285212672
   [junit4]   2> NOTE: All tests run in this JVM: [TestCryptoKeys, TestReRankQParserPlugin, TestPostingsSolrHighlighter, SSLMigrationTest, DistributedQueryComponentOptimizationTest, DistribJoinFromCollectionTest, TestConfig, TestFieldTypeResource, TestHighlightDedupGrouping, TestHashQParserPlugin, DistributedFacetPivotLargeTest, TestCollationFieldDocValues, DocValuesTest, URLClassifyProcessorTest, TestSearchPerf, TestOmitPositions, DocumentAnalysisRequestHandlerTest, DocValuesMissingTest, ConnectionManagerTest, PrimitiveFieldTypeTest, TestSolrConfigHandler, HdfsBasicDistributedZk2Test, BufferStoreTest, TestInitQParser, StandardRequestHandlerTest, MoreLikeThisHandlerTest, SimpleCollectionCreateDeleteTest, TestOverriddenPrefixQueryForCustomFieldType, SpellPossibilityIteratorTest, SolrIndexSplitterTest, TestSchemaSimilarityResource, TestPHPSerializedResponseWriter, PluginInfoTest, TestReversedWildcardFilterFactory, TestBM25SimilarityFactory, TestExpandComponent, TestFastOutputStream, TestSolr4Spatial2, ChaosMonkeySafeLeaderTest, TestRandomDVFaceting, FieldAnalysisRequestHandlerTest, ReturnFieldsTest, MultiThreadedOCPTest, DistributedMLTComponentTest, TestJettySolrRunner, HdfsBasicDistributedZkTest, DistributedDebugComponentTest, JSONWriterTest, TestManagedSchemaFieldResource, TestLRUCache, TestSort, SolrCmdDistributorTest, PeerSyncTest, TriLevelCompositeIdRoutingTest, TestWriterPerf, HighlighterTest, TestBulkSchemaAPI, AlternateDirectoryTest, HttpPartitionTest, ReplicationFactorTest, DistribDocExpirationUpdateProcessorTest, MigrateRouteKeyTest, OverseerRolesTest, TestReplicaProperties]
   [junit4] Completed on J0 in 71.03s, 1 test, 1 failure <<< FAILURES!

[...truncated 667 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 1 failure, 62 ignored (22 assumptions)

Total time: 54 minutes 25 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 224215154 bytes
Compression is 0.0%
Took 42 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2853 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2853/

4 tests failed.
REGRESSION:  org.apache.solr.handler.TestSolrConfigHandlerCloud.test

Error Message:
Could not get expected value  'fomrequest' for path 'params/a' full output: null

Stack Trace:
java.lang.AssertionError: Could not get expected value  'fomrequest' for path 'params/a' full output: null
	at __randomizedtesting.SeedInfo.seed([8476BDE2B05AA48C:C2282381EA6C974]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.core.TestSolrConfigHandler.testForResponseElement(TestSolrConfigHandler.java:406)
	at org.apache.solr.handler.TestSolrConfigHandlerCloud.testReqParams(TestSolrConfigHandlerCloud.java:145)
	at org.apache.solr.handler.TestSolrConfigHandlerCloud.test(TestSolrConfigHandlerCloud.java:78)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:20979/dcva/s/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:20979/dcva/s/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([8476BDE2B05AA48C:C2282381EA6C974]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:41548/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:41548/collection1
	at __randomizedtesting.SeedInfo.seed([8476BDE2B05AA48C:C2282381EA6C974]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:41548/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:45873/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:45873/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([8476BDE2B05AA48C:C2282381EA6C974]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:45873/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9589 lines...]
   [junit4] Suite: org.apache.solr.handler.TestSolrConfigHandlerCloud
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/init-core-data-001
   [junit4]   2> 510884 T1600 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 510884 T1600 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 510889 T1600 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 510890 T1601 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 510990 T1600 oasc.ZkTestServer.run start zk server on port:52711
   [junit4]   2> 510991 T1600 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 510992 T1600 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 511001 T1608 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4145ea2b name:ZooKeeperConnection Watcher:127.0.0.1:52711 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 511001 T1600 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 511002 T1600 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 511002 T1600 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 511004 T1600 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 511005 T1600 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 511006 T1611 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3819a24b name:ZooKeeperConnection Watcher:127.0.0.1:52711/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 511007 T1600 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 511007 T1600 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 511007 T1600 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 511009 T1600 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 511011 T1600 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 511012 T1600 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 511013 T1600 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 511014 T1600 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 511016 T1600 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 511017 T1600 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 511019 T1600 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 511019 T1600 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 511021 T1600 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 511021 T1600 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 511023 T1600 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 511023 T1600 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 511025 T1600 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 511025 T1600 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 511027 T1600 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 511027 T1600 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 511029 T1600 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 511029 T1600 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 511031 T1600 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 511031 T1600 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 511033 T1600 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 511033 T1600 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 511035 T1600 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 511035 T1600 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 512919 T1600 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/control-001/cores/collection1
   [junit4]   2> 512922 T1600 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 512926 T1600 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:52715
   [junit4]   2> 512926 T1600 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=52715, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/tempDir-001/control/data, hostContext=/}
   [junit4]   2> 512927 T1600 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 512927 T1600 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/control-001/'
   [junit4]   2> 512979 T1600 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/control-001/solr.xml
   [junit4]   2> 513044 T1600 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/control-001/cores
   [junit4]   2> 513044 T1600 oasc.CoreContainer.<init> New CoreContainer 1857102062
   [junit4]   2> 513045 T1600 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/control-001/]
   [junit4]   2> 513045 T1600 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/control-001/lib
   [junit4]   2> 513045 T1600 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/control-001/lib).
   [junit4]   2> 513055 T1600 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 513056 T1600 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 513056 T1600 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 513056 T1600 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 513056 T1600 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 513057 T1600 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 513057 T1600 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 513057 T1600 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 513057 T1600 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 513057 T1600 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 513058 T1600 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 513058 T1600 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 513058 T1600 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 513059 T1600 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 513059 T1600 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 513059 T1600 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:52711/solr
   [junit4]   2> 513059 T1600 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 513059 T1600 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 513060 T1600 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 513062 T1623 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5f2a3b02 name:ZooKeeperConnection Watcher:127.0.0.1:52711 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 513063 T1600 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 513063 T1600 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 513065 T1600 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 513066 T1626 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@217cf8a6 name:ZooKeeperConnection Watcher:127.0.0.1:52711/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 513067 T1600 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 513068 T1600 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 513070 T1600 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 513073 T1600 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 513075 T1600 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 513077 T1600 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 513079 T1600 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 513080 T1600 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 513082 T1600 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 513083 T1600 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52715_
   [junit4]   2> 513084 T1600 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:52715_
   [junit4]   2> 513085 T1600 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 513087 T1600 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 513088 T1600 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 513090 T1600 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:52715_
   [junit4]   2> 513090 T1600 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 513092 T1600 oasc.Overseer.start Overseer (id=93559394623946755-127.0.0.1:52715_-n_0000000000) starting
   [junit4]   2> 513094 T1600 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 513100 T1600 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 513101 T1628 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 513101 T1600 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 513101 T1627 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 513106 T1600 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/control-001/cores
   [junit4]   2> 513108 T1600 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {dataDir=data/, config=solrconfig.xml, loadOnStartup=true, collection=control_collection, shard=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/control-001/cores/collection1/, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/control-001/cores/collection1, coreNodeName=, transient=false, name=collection1}
   [junit4]   2> 513109 T1600 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/control-001/cores/collection1/
   [junit4]   2> 513109 T1600 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 513110 T1630 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 513110 T1630 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 513111 T1626 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 513111 T1630 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 513112 T1627 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:52715_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:52715",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1"} current state version: 0
   [junit4]   2> 513113 T1627 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:52715_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:52715",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 513113 T1627 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 513114 T1627 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 513115 T1626 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 514112 T1630 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 514113 T1630 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 514113 T1630 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 514114 T1630 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 514114 T1630 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 514115 T1630 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/control-001/cores/collection1/'
   [junit4]   2> 514136 T1630 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 514138 T1630 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 514146 T1630 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 514181 T1630 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 514258 T1630 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 514259 T1630 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 514271 T1630 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 514848 T1630 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 514862 T1630 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 514865 T1630 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 514876 T1630 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 514880 T1630 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 514894 T1630 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 514895 T1630 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 514896 T1630 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 514896 T1630 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 514897 T1630 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 514897 T1630 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 514897 T1630 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 514897 T1630 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 514897 T1630 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/control-001/cores/collection1/data/
   [junit4]   2> 514898 T1630 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3793258b
   [junit4]   2> 514898 T1630 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/control-001/cores/collection1/data
   [junit4]   2> 514899 T1630 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/control-001/cores/collection1/data/index/
   [junit4]   2> 514899 T1630 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 514899 T1630 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/control-001/cores/collection1/data/index
   [junit4]   2> 514899 T1630 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=42, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.49791655062861495]
   [junit4]   2> 514900 T1630 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5e3445aa lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@39c9cc37),segFN=segments_1,generation=1}
   [junit4]   2> 514900 T1630 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 514905 T1630 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 514905 T1630 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 514905 T1630 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 514905 T1630 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 514906 T1630 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 514906 T1630 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 514906 T1630 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 514906 T1630 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 514906 T1630 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 514927 T1630 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 514929 T1630 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 514930 T1630 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 514932 T1630 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 514936 T1630 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/system,/admin/logging,/admin/properties,/update/json,/update,/update/json/docs,/admin/mbeans,/admin/luke,/replication,standard,/schema,/admin/plugins,/admin/threads,/get,/config,/admin/segments,/admin/ping,/admin/file,/update/csv
   [junit4]   2> 514938 T1630 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 514939 T1630 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 514939 T1630 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 514940 T1630 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 514941 T1630 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 514941 T1630 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=17, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6261984131625574]
   [junit4]   2> 514942 T1630 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5e3445aa lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@39c9cc37),segFN=segments_1,generation=1}
   [junit4]   2> 514942 T1630 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 514942 T1630 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@4b9c5217[collection1] main
   [junit4]   2> 514942 T1630 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 514943 T1630 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 514943 T1630 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 514944 T1630 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 514944 T1630 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 514944 T1630 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 514945 T1630 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 514945 T1630 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 514945 T1630 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 514945 T1630 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 514946 T1631 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4b9c5217[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 514947 T1630 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 514948 T1634 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:52715 collection:control_collection shard:shard1
   [junit4]   2> 514948 T1600 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 514948 T1600 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 514948 T1634 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 514949 T1600 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 514950 T1600 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 514951 T1637 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@421eb33 name:ZooKeeperConnection Watcher:127.0.0.1:52711/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 514952 T1600 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 514952 T1600 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 514953 T1600 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 514954 T1634 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 514956 T1626 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 514956 T1600 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 514956 T1634 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 514957 T1634 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C211 name=collection1 org.apache.solr.core.SolrCore@3f8b5150 url=http://127.0.0.1:52715/collection1 node=127.0.0.1:52715_ C211_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:52715_, state=down, base_url=http://127.0.0.1:52715, core=collection1}
   [junit4]   2> 514957 T1634 C:control_collection S:shard1 C211 P52715 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:52715/collection1/
   [junit4]   2> 514957 T1627 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 514957 T1634 C:control_collection S:shard1 C211 P52715 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 514958 T1634 C:control_collection S:shard1 C211 P52715 oasc.SyncStrategy.syncToMe http://127.0.0.1:52715/collection1/ has no replicas
   [junit4]   2> 514958 T1634 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:52715/collection1/ shard1
   [junit4]   2> 514958 T1634 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 514959 T1637 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 514959 T1626 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 514963 T1626 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 514964 T1627 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:52715",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 515068 T1626 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 515068 T1637 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 515116 T1634 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:52715/collection1/ and leader is http://127.0.0.1:52715/collection1/
   [junit4]   2> 515117 T1634 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:52715
   [junit4]   2> 515117 T1634 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 515117 T1634 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 515117 T1634 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 515119 T1626 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 515120 T1627 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:52715_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:52715",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1"} current state version: 3
   [junit4]   2> 515121 T1627 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:52715_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:52715",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 515224 T1626 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 515224 T1637 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 517255 T1600 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/shard-1-001/cores/collection1
   [junit4]   2> 517259 T1600 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/shard-1-001
   [junit4]   2> 517260 T1600 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 517263 T1600 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:52723
   [junit4]   2> 517263 T1600 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/tempDir-001/jetty1, hostPort=52723, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/shard-1-001/cores, solrconfig=solrconfig.xml, hostContext=/}
   [junit4]   2> 517264 T1600 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 517264 T1600 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/shard-1-001/'
   [junit4]   2> 517333 T1600 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/shard-1-001/solr.xml
   [junit4]   2> 517423 T1600 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/shard-1-001/cores
   [junit4]   2> 517424 T1600 oasc.CoreContainer.<init> New CoreContainer 1094083828
   [junit4]   2> 517424 T1600 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/shard-1-001/]
   [junit4]   2> 517424 T1600 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/shard-1-001/lib
   [junit4]   2> 517429 T1600 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/shard-1-001/lib).
   [junit4]   2> 517448 T1600 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 517449 T1600 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 517449 T1600 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 517449 T1600 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 517449 T1600 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 517450 T1600 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 517450 T1600 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 517451 T1600 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 517451 T1600 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 517451 T1600 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 517451 T1600 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 517452 T1600 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 517453 T1600 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 517453 T1600 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 517453 T1600 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 517454 T1600 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:52711/solr
   [junit4]   2> 517454 T1600 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 517454 T1600 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 517455 T1600 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 517457 T1649 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@72013dcd name:ZooKeeperConnection Watcher:127.0.0.1:52711 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 517457 T1600 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 517458 T1600 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 517460 T1600 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 517462 T1652 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@21bcecc5 name:ZooKeeperConnection Watcher:127.0.0.1:52711/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 517463 T1600 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 517469 T1600 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 518474 T1600 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52723_
   [junit4]   2> 518475 T1600 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:52723_
   [junit4]   2> 518478 T1600 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 518482 T1600 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/shard-1-001/cores
   [junit4]   2> 518485 T1600 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {dataDir=data/, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/shard-1-001/cores/collection1/, config=solrconfig.xml, schema=schema.xml, loadOnStartup=true, name=collection1, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/shard-1-001/cores/collection1, coreNodeName=, collection=collection1, shard=}
   [junit4]   2> 518485 T1600 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/shard-1-001/cores/collection1/
   [junit4]   2> 518486 T1600 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 518487 T1653 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 518487 T1653 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 518488 T1626 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 518488 T1653 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 518489 T1627 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:52723_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:52723",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1"} current state version: 4
   [junit4]   2> 518490 T1627 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:52723_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:52723",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 518490 T1627 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 518490 T1627 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 518491 T1637 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 518491 T1652 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 518491 T1626 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 519488 T1653 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 519489 T1653 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 519490 T1653 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 519490 T1653 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 519491 T1653 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 519491 T1653 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/shard-1-001/cores/collection1/'
   [junit4]   2> 519522 T1653 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 519525 T1653 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 519539 T1653 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 519565 T1653 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 519621 T1653 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 519723 T1653 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 519736 T1653 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 520136 T1653 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 520149 T1653 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 520151 T1653 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 520163 T1653 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 520167 T1653 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 520171 T1653 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 520172 T1653 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 520172 T1653 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 520172 T1653 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 520173 T1653 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 520173 T1653 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 520173 T1653 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 520174 T1653 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 520174 T1653 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 520174 T1653 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3793258b
   [junit4]   2> 520175 T1653 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/shard-1-001/cores/collection1/data
   [junit4]   2> 520176 T1653 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 520176 T1653 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 520176 T1653 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.handler.TestSolrConfigHandlerCloud 8476BDE2B05AA48C-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 520176 T1653 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=42, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.49791655062861495]
   [junit4]   2> 520177 T1653 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@19724e3f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1f6f1726),segFN=segments_1,generation=1}
   [junit4]   2> 520177 T1653 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 520184 T1653 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 520184 T1653 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 520184 T1653 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 520184 T1653 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 520185 T1653 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 520185 T1653 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 520185 T1653 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 520185 T1653 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 520186 T1653 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 520207 T1653 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 520209 T1653 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 520211 T1653 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 520213 T1653 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 520219 T1653 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/system,/admin/logging,/admin/properties,/update/json,/update,/update/json/docs,/admin/mbeans,/admin/luke,/replication,standard,/schema,/admin/plugins,/admin/threads,/get,/config,/admin/segments,/admin/ping,/admin/file,/update/csv
   [junit4]   2> 520221 T1653 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 520222 T1653 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 520223 T1653 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 520224 T1653 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 520224 T1653 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 520225 T1653 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=17, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6261984131625574]
   [junit4]   2> 520226 T1653 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@19724e3f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1f6f1726),segFN=segments_1,generation=1}
   [junit4]   2> 520226 T1653 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 520226 T1653 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@73b512a[collection1] main
   [junit4]   2> 520227 T1653 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 520228 T1653 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 520228 T1653 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 520228 T1653 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 520228 T1653 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 520228 T1653 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 520229 T1653 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 520229 T1653 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 520229 T1653 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 520229 T1653 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 520230 T1654 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@73b512a[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 520231 T1653 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 520231 T1657 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:52723 collection:collection1 shard:shard1
   [junit4]   2> 520232 T1600 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 520232 T1600 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 520232 T1657 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 520237 T1657 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 520239 T1626 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 520239 T1657 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 520239 T1657 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C212 name=collection1 org.apache.solr.core.SolrCore@3987ad3c url=http://127.0.0.1:52723/collection1 node=127.0.0.1:52723_ C212_STATE=coll:collection1 core:collection1 props:{node_name=127.0.0.1:52723_, state=down, base_url=http://127.0.0.1:52723, core=collection1}
   [junit4]   2> 520240 T1657 C:collection1 S:shard1 C212 P52723 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:52723/collection1/
   [junit4]   2

[...truncated too long message...]

 4 connections to: http://127.0.0.1:45851/
   [junit4]   2> 1621554 T3191 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:45837/
   [junit4]   2> 1621554 T3191 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 11 connections to: http://127.0.0.1:45873/
   [junit4]   2> 1621555 T3191 C:control_collection S:shard1 c:collection1 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 1621555 T3191 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:45825/
   [junit4]   2> 1621555 T3191 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:45851/
   [junit4]   2> 1621556 T3191 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:45837/
   [junit4]   2> 1621556 T3191 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:45873/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=8476BDE2B05AA48C -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ko_KR -Dtests.timezone=Pacific/Guadalcanal -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   76.7s J1 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:45873/repfacttest_c8n_1x3_shard1_replica1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([8476BDE2B05AA48C:C2282381EA6C974]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:45873/repfacttest_c8n_1x3_shard1_replica1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 48 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 53 more
   [junit4]   2> 1621571 T3191 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 8476BDE2B05AA48C-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {intDefault=Lucene50(blocksize=128), multiDefault=PostingsFormat(name=LuceneFixedGap), timestamp=PostingsFormat(name=LuceneFixedGap), id=FST50, a_t=PostingsFormat(name=LuceneFixedGap), range_facet_l=PostingsFormat(name=LuceneFixedGap), text=PostingsFormat(name=LuceneVarGapFixedInterval), _version_=Lucene50(blocksize=128)}, docValues:{timestamp=DocValuesFormat(name=Direct)}, sim=RandomSimilarityProvider(queryNorm=false,coord=yes): {}, locale=ko_KR, timezone=Pacific/Guadalcanal
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=82615000,total=300941312
   [junit4]   2> NOTE: All tests run in this JVM: [PeerSyncTest, TestDistribDocBasedVersion, ConnectionManagerTest, ExternalFileFieldSortTest, HdfsUnloadDistributedZkTest, MinimalSchemaTest, PolyFieldTest, DistributedSpellCheckComponentTest, ParsingFieldUpdateProcessorsTest, DeleteLastCustomShardedReplicaTest, UnloadDistributedZkTest, LeaderFailoverAfterPartitionTest, CoreAdminRequestStatusTest, TestStressRecovery, TestPhraseSuggestions, TestDefaultSimilarityFactory, DistributedFacetPivotLongTailTest, TestUpdate, TestUtils, SynonymTokenizerTest, TestPerFieldSimilarity, RollingRestartTest, TestStressLucene, TestFaceting, ChaosMonkeyNothingIsSafeTest, TestLRUCache, SolrPluginUtilsTest, TestFieldCollectionResource, CachingDirectoryFactoryTest, TestCloudInspectUtil, HighlighterMaxOffsetTest, TestTolerantSearch, DisMaxRequestHandlerTest, TestSchemaManager, TestOverriddenPrefixQueryForCustomFieldType, SolrCmdDistributorTest, SignatureUpdateProcessorFactoryTest, DeleteShardTest, FieldAnalysisRequestHandlerTest, TestCloudManagedSchema, TestCSVResponseWriter, SolrIndexConfigTest, TestReloadAndDeleteDocs, TestPseudoReturnFields, TestDFRSimilarityFactory, TestOmitPositions, TestUniqueKeyFieldResource, HdfsBasicDistributedZk2Test, TestFieldTypeCollectionResource, TestSurroundQueryParser, HdfsBasicDistributedZkTest, DistribCursorPagingTest, TestManagedResourceStorage, MBeansHandlerTest, DocValuesMultiTest, RecoveryAfterSoftCommitTest, ShardSplitTest, LeaderInitiatedRecoveryOnCommitTest, ReplicationFactorTest]
   [junit4] Completed on J1 in 76.79s, 1 test, 1 error <<< FAILURES!

[...truncated 714 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 1 failure, 62 ignored (22 assumptions)

Total time: 56 minutes 22 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 219962049 bytes
Compression is 0.0%
Took 46 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2852 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2852/

4 tests failed.
REGRESSION:  org.apache.solr.handler.component.DistributedMLTComponentTest.test

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:45389/fv/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:45389/fv/collection1
	at __randomizedtesting.SeedInfo.seed([5E5EECC7344C5537:D60AD31D9AB038CF]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:566)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:943)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:558)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:606)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:588)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:567)
	at org.apache.solr.handler.component.DistributedMLTComponentTest.test(DistributedMLTComponentTest.java:126)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:442)
	at sun.security.ssl.InputRecord.read(InputRecord.java:480)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:927)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:884)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:102)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 51 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:13315/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:13315/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([5E5EECC7344C5537:D60AD31D9AB038CF]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:15593/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:15593/collection1
	at __randomizedtesting.SeedInfo.seed([5E5EECC7344C5537:D60AD31D9AB038CF]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:15593/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:24347/epm/f/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:24347/epm/f/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([5E5EECC7344C5537:D60AD31D9AB038CF]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:24347/epm/f/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9481 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/init-core-data-001
   [junit4]   2> 595214 T3205 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /epm/f
   [junit4]   2> 595219 T3205 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 595220 T3206 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 595320 T3205 oasc.ZkTestServer.run start zk server on port:34419
   [junit4]   2> 595321 T3205 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 595322 T3205 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 595325 T3213 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@75ccd577 name:ZooKeeperConnection Watcher:127.0.0.1:34419 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 595326 T3205 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 595326 T3205 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 595326 T3205 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 595329 T3205 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 595330 T3205 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 595331 T3216 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@778155c5 name:ZooKeeperConnection Watcher:127.0.0.1:34419/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 595331 T3205 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 595332 T3205 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 595332 T3205 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 595334 T3205 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 595335 T3205 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 595337 T3205 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 595338 T3205 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 595339 T3205 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 595342 T3205 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 595342 T3205 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 595344 T3205 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 595345 T3205 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 595347 T3205 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 595347 T3205 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 595349 T3205 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 595349 T3205 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 595351 T3205 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 595352 T3205 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 595353 T3205 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 595354 T3205 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 595356 T3205 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 595356 T3205 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 595358 T3205 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 595358 T3205 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 595360 T3205 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 595361 T3205 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 595362 T3205 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 595363 T3205 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 596498 T3205 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/control-001/cores/collection1
   [junit4]   2> 596502 T3205 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 596506 T3205 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:39780
   [junit4]   2> 596507 T3205 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/control-001/cores, hostPort=24347, hostContext=/epm/f}
   [junit4]   2> 596507 T3205 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 596507 T3205 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/control-001/'
   [junit4]   2> 596551 T3205 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/control-001/solr.xml
   [junit4]   2> 596603 T3205 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/control-001/cores
   [junit4]   2> 596603 T3205 oasc.CoreContainer.<init> New CoreContainer 2104915078
   [junit4]   2> 596604 T3205 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/control-001/]
   [junit4]   2> 596604 T3205 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/control-001/lib
   [junit4]   2> 596605 T3205 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/control-001/lib).
   [junit4]   2> 596614 T3205 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 596615 T3205 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 596615 T3205 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 596615 T3205 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 596615 T3205 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 596616 T3205 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 596616 T3205 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 596616 T3205 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 596616 T3205 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 596617 T3205 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 596617 T3205 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 596617 T3205 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 596618 T3205 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 596618 T3205 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 596618 T3205 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 596618 T3205 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:34419/solr
   [junit4]   2> 596619 T3205 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 596619 T3205 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 596620 T3205 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 596621 T3228 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@75b907e9 name:ZooKeeperConnection Watcher:127.0.0.1:34419 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 596622 T3205 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 596623 T3205 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 596625 T3205 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 596626 T3231 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@75f4d7f0 name:ZooKeeperConnection Watcher:127.0.0.1:34419/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 596627 T3205 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 596628 T3205 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 596631 T3205 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 596633 T3205 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 596635 T3205 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 596637 T3205 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 596640 T3205 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 596641 T3205 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 596643 T3205 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 596644 T3205 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:24347_epm%2Ff
   [junit4]   2> 596645 T3205 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:24347_epm%2Ff
   [junit4]   2> 596646 T3205 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 596648 T3205 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 596649 T3205 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 596651 T3205 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:24347_epm%2Ff
   [junit4]   2> 596651 T3205 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 596653 T3205 oasc.Overseer.start Overseer (id=93558450850234371-127.0.0.1:24347_epm%2Ff-n_0000000000) starting
   [junit4]   2> 596655 T3205 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 596662 T3205 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 596662 T3233 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 596662 T3205 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 596663 T3232 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 596666 T3205 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/control-001/cores
   [junit4]   2> 596668 T3205 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/control-001/cores/collection1/, schema=schema.xml, coreNodeName=, name=collection1, shard=, loadOnStartup=true, collection=control_collection, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/control-001/cores/collection1, transient=false, config=solrconfig.xml, dataDir=data/}
   [junit4]   2> 596669 T3205 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/control-001/cores/collection1/
   [junit4]   2> 596669 T3205 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 596670 T3235 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 596670 T3235 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 596671 T3231 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 596671 T3235 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 596672 T3232 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:24347/epm/f",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:24347_epm%2Ff",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null} current state version: 0
   [junit4]   2> 596673 T3232 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:24347/epm/f",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:24347_epm%2Ff",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 596673 T3232 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 596673 T3232 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 596674 T3231 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 597672 T3235 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 597673 T3235 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 597673 T3235 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 597674 T3235 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 597674 T3235 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 597675 T3235 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/control-001/cores/collection1/'
   [junit4]   2> 597687 T3235 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 597689 T3235 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 597697 T3235 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 597722 T3235 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 597756 T3235 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 597757 T3235 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 597766 T3235 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 598035 T3235 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 598046 T3235 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 598048 T3235 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 598061 T3235 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 598064 T3235 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 598067 T3235 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 598068 T3235 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 598068 T3235 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 598068 T3235 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 598069 T3235 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 598069 T3235 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 598069 T3235 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 598069 T3235 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 598070 T3235 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/control-001/cores/collection1/data/
   [junit4]   2> 598070 T3235 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@167f3e42
   [junit4]   2> 598071 T3235 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/control-001/cores/collection1/data
   [junit4]   2> 598071 T3235 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/control-001/cores/collection1/data/index/
   [junit4]   2> 598071 T3235 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 598071 T3235 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/control-001/cores/collection1/data/index
   [junit4]   2> 598072 T3235 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=23, maxMergeAtOnceExplicit=35, maxMergedSegmentMB=92.7392578125, floorSegmentMB=1.84375, forceMergeDeletesPctAllowed=16.56549626313214, segmentsPerTier=29.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.2992785701384511
   [junit4]   2> 598072 T3235 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2bf1de72 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@144324e8),segFN=segments_1,generation=1}
   [junit4]   2> 598073 T3235 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 598078 T3235 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 598078 T3235 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 598078 T3235 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 598079 T3235 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 598079 T3235 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 598079 T3235 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 598079 T3235 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 598080 T3235 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 598080 T3235 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 598094 T3235 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 598096 T3235 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 598097 T3235 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 598099 T3235 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 598104 T3235 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/properties,/replication,/update/csv,/admin/file,/schema,/admin/plugins,/admin/mbeans,/update,/admin/ping,/admin/segments,/admin/threads,/admin/logging,/get,/update/json,/admin/luke,/update/json/docs,standard,/admin/system,/config
   [junit4]   2> 598105 T3235 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 598107 T3235 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 598107 T3235 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 598108 T3235 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 598108 T3235 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 598109 T3235 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=44, maxMergeAtOnceExplicit=18, maxMergedSegmentMB=43.01171875, floorSegmentMB=0.6806640625, forceMergeDeletesPctAllowed=6.998341864313913, segmentsPerTier=28.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 598109 T3235 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2bf1de72 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@144324e8),segFN=segments_1,generation=1}
   [junit4]   2> 598110 T3235 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 598110 T3235 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@59f776ce[collection1] main
   [junit4]   2> 598110 T3235 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 598111 T3235 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 598111 T3235 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 598111 T3235 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 598112 T3235 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 598112 T3235 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 598112 T3235 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 598113 T3235 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 598113 T3235 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 598113 T3235 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 598114 T3236 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@59f776ce[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 598114 T3235 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 598115 T3239 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:24347/epm/f collection:control_collection shard:shard1
   [junit4]   2> 598115 T3205 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 598115 T3205 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 598116 T3239 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 598118 T3205 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 598119 T3205 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 598120 T3239 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 598120 T3243 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@313607ce name:ZooKeeperConnection Watcher:127.0.0.1:34419/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 598120 T3205 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 598121 T3205 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 598121 T3205 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 598122 T3231 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 598122 T3239 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 598122 T3239 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C611 name=collection1 org.apache.solr.core.SolrCore@4fffb9dd url=http://127.0.0.1:24347/epm/f/collection1 node=127.0.0.1:24347_epm%2Ff C611_STATE=coll:control_collection core:collection1 props:{core=collection1, base_url=http://127.0.0.1:24347/epm/f, node_name=127.0.0.1:24347_epm%2Ff, state=down}
   [junit4]   2> 598122 T3239 C:control_collection S:shard1 C611 P24347 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:24347/epm/f/collection1/
   [junit4]   2> 598123 T3232 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 598123 T3239 C:control_collection S:shard1 C611 P24347 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 598123 T3239 C:control_collection S:shard1 C611 P24347 oasc.SyncStrategy.syncToMe http://127.0.0.1:24347/epm/f/collection1/ has no replicas
   [junit4]   2> 598123 T3239 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:24347/epm/f/collection1/ shard1
   [junit4]   2> 598123 T3239 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 598124 T3205 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 598124 T3205 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 598125 T3205 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 598125 T3205 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 598126 T3246 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@62fdf315 name:ZooKeeperConnection Watcher:127.0.0.1:34419/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 598126 T3205 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 598127 T3205 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 598127 T3231 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 598128 T3232 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:24347/epm/f",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 1
   [junit4]   2> 598131 T3232 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 1
   [junit4]   2> 598131 T3232 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 598132 T3231 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 598132 T3243 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 598135 T3231 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 598136 T3232 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 598137 T3243 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 598137 T3231 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 598178 T3239 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:24347/epm/f/collection1/ and leader is http://127.0.0.1:24347/epm/f/collection1/
   [junit4]   2> 598179 T3239 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:24347/epm/f
   [junit4]   2> 598179 T3239 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 598179 T3239 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 598179 T3239 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 598180 T3231 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 598181 T3232 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:24347/epm/f",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:24347_epm%2Ff",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1"} current state version: 3
   [junit4]   2> 598181 T3232 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:24347/epm/f",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:24347_epm%2Ff",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1"}
   [junit4]   2> 598285 T3231 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 598285 T3243 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 599402 T3205 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/shard-1-001/cores/collection1
   [junit4]   2> 599405 T3205 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/shard-1-001
   [junit4]   2> 599405 T3205 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 599408 T3205 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:64156
   [junit4]   2> 599408 T3205 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/shard-1-001/cores, hostContext=/epm/f, hostPort=40574, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/tempDir-001/jetty1, solrconfig=solrconfig.xml}
   [junit4]   2> 599408 T3205 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 599409 T3205 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/shard-1-001/'
   [junit4]   2> 599452 T3205 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/shard-1-001/solr.xml
   [junit4]   2> 599507 T3205 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/shard-1-001/cores
   [junit4]   2> 599507 T3205 oasc.CoreContainer.<init> New CoreContainer 499084204
   [junit4]   2> 599507 T3205 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/shard-1-001/]
   [junit4]   2> 599508 T3205 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/shard-1-001/lib
   [junit4]   2> 599508 T3205 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/shard-1-001/lib).
   [junit4]   2> 599519 T3205 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 599519 T3205 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 599519 T3205 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 599519 T3205 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 599520 T3205 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 599520 T3205 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 599520 T3205 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 599520 T3205 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 599521 T3205 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 599521 T3205 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 599521 T3205 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 599522 T3205 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 599522 T3205 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 599522 T3205 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 599522 T3205 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 599523 T3205 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:34419/solr
   [junit4]   2> 599523 T3205 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 599523 T3205 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 599524 T3205 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 599525 T3258 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4e3121d name:ZooKeeperConnection Watcher:127.0.0.1:34419 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 599526 T3205 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 599527 T3205 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 599529 T3205 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 599530 T3261 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3eadf24d name:ZooKeeperConnection Watcher:127.0.0.1:34419/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 599530 T3205 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 599535 T3205 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 600540 T3205 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40574_epm%2Ff
   [junit4]   2> 600541 T3205 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:40574_epm%2Ff
   [junit4]   2> 600544 T3205 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 600546 T3205 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/shard-1-001/cores
   [junit4]   2> 600548 T3205 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {transient=false, loadOnStartup=true, collection=collection1, schema=schema.xml, shard=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/shard-1-001/cores/collection1/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/shard-1-001/cores/collection1, dataDir=data/, config=solrconfig.xml, coreNodeName=, name=collection1}
   [junit4]   2> 600548 T3205 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/shard-1-001/cores/collection1/
   [junit4]   2> 600548 T3205 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 600550 T3262 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 600550 T3262 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 600552 T3231 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 600552 T3262 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 600552 T3262 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 600553 T3232 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:40574/epm/f",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:40574_epm%2Ff",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null} current state version: 4
   [junit4]   2> 600553 T3262 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 600553 T3232 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:40574/epm/f",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "node_name":"127.0.0.1:40574_epm%2Ff",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 600553 T3232 oasco.ReplicaMutator.updateState Collection already exists with numShards=3
   [junit4]   2> 600554 T3232 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 600554 T3262 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 600554 T3262 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 600555 T3232 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 600555 T3261 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 600556 T3261 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 601554 T3262 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 601555 T3262 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 601555 T3262 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 601556 T3262 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 601556 T3262 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 601556 T3262 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/shard-1-001/cores/collection1/'
   [junit4]   2> 601569 T3262 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 601571 T3262 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 601579 T3262 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 601594 T3262 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 601628 T3262 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 601629 T3262 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 601638 T3262 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 601907 T3262 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 601918 T3262 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 601920 T3262 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 601932 T3262 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 601935 T3262 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 601938 T3262 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 601939 T3262 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 601939 T3262 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 601939 T3262 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 601940 T3262 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 601940 T3262 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 601940 T3262 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 601940 T3262 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 601941 T3262 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 601941 T3262 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@167f3e42
   [junit4]   2> 601942 T3262 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/shard-1-001/cores/collection1/data
   [junit4]   2> 601942 T3262 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 601942 T3262 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 601942 T3262 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 5E5EECC7344C5537-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 601943 T3262 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=23, maxMergeAtOnceExplicit=35, maxMergedSegmentMB=92.7392578125, floorSegmentMB=1.84375, forceMergeDeletesPctAllowed=16.56549626313214, segmentsPerTier=29.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.2992785701384511
   [junit4]   2> 601943 T3262 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1944f557 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7e472fb6),segFN=segments_1,generation=1}
   [junit4]   2> 601944 T3262 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 601950 T3262 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 601951 T3262 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 601951 T3262 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 601951 T3262 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 601952 T3262 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 601952 T3262 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 601952 T3262 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 601952 T3262 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 601953 T3262 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 601968 T3262 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 601969 T3262 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 601971 T3262 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 601973 T3262 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 601980 T3262 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/properties,/replication,/update/csv,/admin/file,/schema,/admin/plugins,/admin/mbeans,/update,/admin/ping,/admin/segments,/admin/threads,/admin/logging,/get,/update/json,/admin/luke,/update/json/docs,standard,/admin/system,/config
   [junit4]   2> 601982 T3262 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 601983 T3262 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 601983 T3262 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2>

[...truncated too long message...]

ing to 127.0.0.1:13274 13274
   [junit4]   2> 1586917 T2940 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:13274 13274
   [junit4]   2> 1586918 T2940 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/aliases.json
   [junit4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 		4	/solr/collections/collection1/state.json
   [junit4]   2> 		3	/solr/collections/c8n_1x3_commits/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 		4	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1586918 T2939 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:13315/
   [junit4]   2> 1586919 T2939 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:13360/
   [junit4]   2> 1586919 T2939 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:13299/
   [junit4]   2> 1586919 T2939 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:13277/
   [junit4]   2> 1586920 T2939 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 12 connections to: http://127.0.0.1:13293/
   [junit4]   2> 1586920 T2939 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:13315/
   [junit4]   2> 1586921 T2939 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:13360/
   [junit4]   2> 1586921 T2939 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:13299/
   [junit4]   2> 1586921 T2939 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:13277/
   [junit4]   2> 1586921 T2939 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:13293/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderInitiatedRecoveryOnCommitTest -Dtests.method=test -Dtests.seed=5E5EECC7344C5537 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=lv_LV -Dtests.timezone=America/Pangnirtung -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   80.1s J1 | LeaderInitiatedRecoveryOnCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:13315/c8n_1x3_commits_shard1_replica2
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([5E5EECC7344C5537:D60AD31D9AB038CF]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 48 more
   [junit4]   2> 1586951 T2939 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 5E5EECC7344C5537-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=DefaultSimilarity, locale=lv_LV, timezone=America/Pangnirtung
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=171440336,total=316145664
   [junit4]   2> NOTE: All tests run in this JVM: [TestValueSourceCache, TestLeaderElectionZkExpiry, CursorPagingTest, TestFieldTypeCollectionResource, SpellPossibilityIteratorTest, TestUtils, TestManagedSynonymFilterFactory, DOMUtilTest, TestSolrCoreProperties, TestBM25SimilarityFactory, HdfsCollectionsAPIDistributedZkTest, AnalysisAfterCoreReloadTest, ZkControllerTest, TestStressReorder, TestComplexPhraseQParserPlugin, SpellCheckComponentTest, DistributedQueueTest, QueryResultKeyTest, HdfsChaosMonkeySafeLeaderTest, AutoCommitTest, DistributedIntervalFacetingTest, TestDistributedGrouping, DistributedDebugComponentTest, TestRemoteStreaming, TestMergePolicyConfig, TestDocumentBuilder, TestRecoveryHdfs, TestGroupingSearch, PreAnalyzedUpdateProcessorTest, XmlUpdateRequestHandlerTest, TestHashPartitioner, BasicFunctionalityTest, TestIntervalFaceting, TestDynamicLoading, TestConfigOverlay, SynonymTokenizerTest, TestAnalyzeInfixSuggestions, SpatialRPTFieldTypeTest, TestBulkSchemaConcurrent, ReturnFieldsTest, IndexBasedSpellCheckerTest, TestPerFieldSimilarity, TestStressUserVersions, SpatialHeatmapFacetsTest, TestBulkSchemaAPI, AssignTest, LeaderFailoverAfterPartitionTest, TermsComponentTest, DistribDocExpirationUpdateProcessorTest, UniqFieldsUpdateProcessorFactoryTest, ChaosMonkeyNothingIsSafeTest, CSVRequestHandlerTest, CoreAdminHandlerTest, InfoHandlerTest, WordBreakSolrSpellCheckerTest, TestExactSharedStatsCache, DistributedTermsComponentTest, ExternalCollectionsTest, SegmentsInfoRequestHandlerTest, LeaderInitiatedRecoveryOnCommitTest]
   [junit4] Completed on J1 in 80.10s, 1 test, 1 error <<< FAILURES!

[...truncated 738 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 4 errors, 62 ignored (22 assumptions)

Total time: 54 minutes 37 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 218346238 bytes
Compression is 0.0%
Took 42 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2851 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2851/

4 tests failed.
REGRESSION:  org.apache.solr.cloud.RecoveryZkTest.test

Error Message:
shard1 is not consistent.  Got 837 from http://127.0.0.1:34371/collection1lastClient and got 209 from http://127.0.0.1:34415/collection1

Stack Trace:
java.lang.AssertionError: shard1 is not consistent.  Got 837 from http://127.0.0.1:34371/collection1lastClient and got 209 from http://127.0.0.1:34415/collection1
	at __randomizedtesting.SeedInfo.seed([F22FA1F94A30AA2D:7A7B9E23E4CCC7D5]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.RecoveryZkTest.test(RecoveryZkTest.java:123)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:46524/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:46524/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([F22FA1F94A30AA2D:7A7B9E23E4CCC7D5]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([F22FA1F94A30AA2D:7A7B9E23E4CCC7D5]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([F22FA1F94A30AA2D:7A7B9E23E4CCC7D5]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9387 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/init-core-data-001
   [junit4]   2> 180859 T824 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /iiknr/
   [junit4]   2> 180869 T824 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 180870 T825 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 180970 T824 oasc.ZkTestServer.run start zk server on port:41933
   [junit4]   2> 180971 T824 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 180972 T824 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 180976 T832 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5b1532ed name:ZooKeeperConnection Watcher:127.0.0.1:41933 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 180976 T824 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 180977 T824 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 180977 T824 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 180980 T824 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 180981 T824 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 180983 T835 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1c06dda5 name:ZooKeeperConnection Watcher:127.0.0.1:41933/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 180983 T824 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 180984 T824 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 180984 T824 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 180986 T824 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 180987 T824 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 180988 T824 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 180990 T824 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 180991 T824 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 180993 T824 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 180993 T824 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 180995 T824 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 180996 T824 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 180997 T824 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 180997 T824 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 180999 T824 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 180999 T824 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 181000 T824 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 181001 T824 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 181002 T824 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 181003 T824 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 181004 T824 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 181005 T824 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 181006 T824 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 181007 T824 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 181008 T824 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 181009 T824 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 181010 T824 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 181010 T824 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 181962 T824 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/control-001/cores/collection1
   [junit4]   2> 181966 T824 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 181968 T824 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:41937
   [junit4]   2> 181968 T824 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/control-001/cores, hostContext=/iiknr, hostPort=41936, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/tempDir-001/control/data}
   [junit4]   2> 181969 T824 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 181969 T824 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/control-001/'
   [junit4]   2> 181999 T824 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/control-001/solr.xml
   [junit4]   2> 182033 T824 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/control-001/cores
   [junit4]   2> 182034 T824 oasc.CoreContainer.<init> New CoreContainer 1122098765
   [junit4]   2> 182034 T824 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/control-001/]
   [junit4]   2> 182034 T824 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/control-001/lib
   [junit4]   2> 182035 T824 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/control-001/lib).
   [junit4]   2> 182044 T824 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 182044 T824 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 182045 T824 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 182045 T824 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 182045 T824 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 182045 T824 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 182046 T824 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 182046 T824 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 182046 T824 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 182046 T824 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 182047 T824 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 182047 T824 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 182048 T824 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 182048 T824 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 182048 T824 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 182048 T824 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:41933/solr
   [junit4]   2> 182049 T824 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 182049 T824 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 182050 T824 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 182052 T847 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2e3a4857 name:ZooKeeperConnection Watcher:127.0.0.1:41933 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 182053 T824 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 182053 T824 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 182055 T824 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 182057 T850 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@25199335 name:ZooKeeperConnection Watcher:127.0.0.1:41933/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 182057 T824 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 182059 T824 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 182061 T824 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 182062 T824 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 182064 T824 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 182066 T824 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 182068 T824 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 182070 T824 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 182071 T824 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 182072 T824 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41936_iiknr
   [junit4]   2> 182073 T824 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:41936_iiknr
   [junit4]   2> 182075 T824 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 182076 T824 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 182077 T824 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 182079 T824 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:41936_iiknr
   [junit4]   2> 182079 T824 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 182080 T824 oasc.Overseer.start Overseer (id=93557476896997379-127.0.0.1:41936_iiknr-n_0000000000) starting
   [junit4]   2> 182082 T824 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 182088 T824 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 182088 T852 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 182089 T851 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 182088 T824 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 182092 T824 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/control-001/cores
   [junit4]   2> 182094 T824 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {dataDir=data/, transient=false, shard=, loadOnStartup=true, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/control-001/cores/collection1, config=solrconfig.xml, coreNodeName=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/control-001/cores/collection1/, collection=control_collection, name=collection1, schema=schema.xml}
   [junit4]   2> 182095 T824 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/control-001/cores/collection1/
   [junit4]   2> 182095 T824 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 182096 T854 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 182097 T854 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 182098 T854 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 182098 T850 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 182099 T851 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41936/iiknr",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:41936_iiknr",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null} current state version: 0
   [junit4]   2> 182099 T851 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41936/iiknr",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:41936_iiknr",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 182099 T851 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 182100 T851 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 182101 T850 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 183098 T854 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 183099 T854 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 183100 T854 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 183100 T854 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 183100 T854 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 183101 T854 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/control-001/cores/collection1/'
   [junit4]   2> 183115 T854 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 183117 T854 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 183127 T854 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 183144 T854 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 183184 T854 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 183186 T854 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 183194 T854 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 183519 T854 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 183533 T854 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 183536 T854 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 183549 T854 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 183553 T854 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 183557 T854 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 183558 T854 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 183559 T854 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 183559 T854 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 183560 T854 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 183560 T854 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 183560 T854 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 183560 T854 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 183561 T854 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/control-001/cores/collection1/data/
   [junit4]   2> 183561 T854 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@78c8ec7e
   [junit4]   2> 183562 T854 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/control-001/cores/collection1/data
   [junit4]   2> 183562 T854 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/control-001/cores/collection1/data/index/
   [junit4]   2> 183562 T854 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 183563 T854 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/control-001/cores/collection1/data/index
   [junit4]   2> 183563 T854 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=29, maxMergeAtOnceExplicit=33, maxMergedSegmentMB=86.779296875, floorSegmentMB=0.7470703125, forceMergeDeletesPctAllowed=22.320311402449484, segmentsPerTier=19.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 183564 T854 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3fb87a10 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6855d11e),segFN=segments_1,generation=1}
   [junit4]   2> 183564 T854 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 183569 T854 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 183569 T854 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 183569 T854 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 183570 T854 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 183570 T854 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 183570 T854 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 183570 T854 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 183571 T854 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 183571 T854 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 183593 T854 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 183595 T854 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 183598 T854 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 183600 T854 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 183604 T854 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/luke,/admin/file,/replication,/update,/admin/logging,standard,/admin/segments,/admin/plugins,/admin/ping,/admin/properties,/update/json/docs,/admin/system,/schema,/config,/admin/threads,/get,/update/csv,/update/json,/admin/mbeans
   [junit4]   2> 183608 T854 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 183611 T854 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 183614 T854 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 183619 T854 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 183619 T854 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 183621 T854 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=48, maxMergeAtOnceExplicit=20, maxMergedSegmentMB=72.1044921875, floorSegmentMB=1.9287109375, forceMergeDeletesPctAllowed=10.126703683436897, segmentsPerTier=15.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 183622 T854 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3fb87a10 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6855d11e),segFN=segments_1,generation=1}
   [junit4]   2> 183622 T854 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 183623 T854 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@3c78fa09[collection1] main
   [junit4]   2> 183624 T854 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 183625 T854 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 183626 T854 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 183626 T854 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 183627 T854 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 183627 T854 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 183628 T854 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 183629 T854 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 183629 T854 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 183629 T854 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 183634 T855 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3c78fa09[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 183635 T854 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 183636 T858 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:41936/iiknr collection:control_collection shard:shard1
   [junit4]   2> 183637 T824 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 183638 T824 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 183639 T858 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 183644 T824 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 183645 T858 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 183646 T824 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 183648 T862 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@17b29b34 name:ZooKeeperConnection Watcher:127.0.0.1:41933/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 183649 T850 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 183649 T824 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 183650 T851 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 183649 T858 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 183650 T858 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 183650 T824 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 183651 T850 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> ASYNC  NEW_CORE C97 name=collection1 org.apache.solr.core.SolrCore@736823b url=http://127.0.0.1:41936/iiknr/collection1 node=127.0.0.1:41936_iiknr C97_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:41936/iiknr, core=collection1, node_name=127.0.0.1:41936_iiknr, state=down}
   [junit4]   2> 183651 T858 C:control_collection S:shard1 C97 P41936 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:41936/iiknr/collection1/
   [junit4]   2> 183651 T824 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 183652 T858 C:control_collection S:shard1 C97 P41936 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 183653 T858 C:control_collection S:shard1 C97 P41936 oasc.SyncStrategy.syncToMe http://127.0.0.1:41936/iiknr/collection1/ has no replicas
   [junit4]   2> 183654 T858 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:41936/iiknr/collection1/ shard1
   [junit4]   2> 183654 T858 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 183655 T824 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 183657 T824 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 183658 T824 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 183659 T824 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 183661 T865 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4957b319 name:ZooKeeperConnection Watcher:127.0.0.1:41933/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 183662 T824 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 183664 T824 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 183677 T850 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 183678 T851 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41936/iiknr",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 183694 T850 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 183696 T851 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 183697 T851 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 183698 T862 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 183700 T850 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 183729 T858 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:41936/iiknr/collection1/ and leader is http://127.0.0.1:41936/iiknr/collection1/
   [junit4]   2> 183729 T858 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:41936/iiknr
   [junit4]   2> 183729 T858 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 183730 T858 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 183730 T858 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 183732 T850 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 183733 T851 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41936/iiknr",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:41936_iiknr",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1"} current state version: 3
   [junit4]   2> 183734 T851 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41936/iiknr",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:41936_iiknr",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1"}
   [junit4]   2> 183735 T851 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 183738 T850 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 183738 T862 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 183842 T862 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 183842 T850 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 185269 T824 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/shard-1-001/cores/collection1
   [junit4]   2> 185271 T824 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/shard-1-001
   [junit4]   2> 185272 T824 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 185275 T824 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:41959
   [junit4]   2> 185275 T824 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/iiknr, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/shard-1-001/cores, hostPort=41958, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/tempDir-001/jetty1, solrconfig=solrconfig.xml}
   [junit4]   2> 185276 T824 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 185276 T824 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/shard-1-001/'
   [junit4]   2> 185306 T824 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/shard-1-001/solr.xml
   [junit4]   2> 185342 T824 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/shard-1-001/cores
   [junit4]   2> 185343 T824 oasc.CoreContainer.<init> New CoreContainer 27349232
   [junit4]   2> 185343 T824 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/shard-1-001/]
   [junit4]   2> 185343 T824 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/shard-1-001/lib
   [junit4]   2> 185344 T824 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/shard-1-001/lib).
   [junit4]   2> 185353 T824 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 185354 T824 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 185354 T824 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 185354 T824 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 185355 T824 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 185355 T824 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 185355 T824 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 185355 T824 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 185356 T824 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 185356 T824 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 185356 T824 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 185357 T824 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 185357 T824 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 185357 T824 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 185358 T824 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 185358 T824 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:41933/solr
   [junit4]   2> 185358 T824 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 185358 T824 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 185370 T824 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 185372 T877 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5d15a5b8 name:ZooKeeperConnection Watcher:127.0.0.1:41933 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 185373 T824 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 185373 T824 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 185375 T824 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 185377 T880 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6a6e3624 name:ZooKeeperConnection Watcher:127.0.0.1:41933/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 185377 T824 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 185380 T824 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 186384 T824 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41958_iiknr
   [junit4]   2> 186385 T824 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:41958_iiknr
   [junit4]   2> 186388 T824 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 186391 T824 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/shard-1-001/cores
   [junit4]   2> 186393 T824 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {loadOnStartup=true, transient=false, name=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/shard-1-001/cores/collection1/, collection=collection1, schema=schema.xml, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/shard-1-001/cores/collection1, shard=, coreNodeName=, dataDir=data/}
   [junit4]   2> 186394 T824 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/shard-1-001/cores/collection1/
   [junit4]   2> 186394 T824 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 186397 T881 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 186397 T881 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 186398 T850 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 186398 T881 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 186399 T881 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 186399 T851 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41958/iiknr",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:41958_iiknr",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null} current state version: 5
   [junit4]   2> 186399 T881 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 186400 T851 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41958/iiknr",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:41958_iiknr",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 186400 T851 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 186400 T851 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 186400 T881 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 186400 T881 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 186401 T851 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 186402 T880 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 186402 T880 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 187401 T881 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 187403 T881 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 187403 T881 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 187404 T881 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 187405 T881 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 187405 T881 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/shard-1-001/cores/collection1/'
   [junit4]   2> 187427 T881 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 187431 T881 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 187449 T881 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 187483 T881 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 187572 T881 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 187578 T881 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 187592 T881 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 188041 T881 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 188059 T881 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 188062 T881 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 188077 T881 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 188082 T881 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 188087 T881 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 188087 T881 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 188088 T881 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 188088 T881 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 188089 T881 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 188089 T881 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 188090 T881 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 188090 T881 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 188090 T881 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 188091 T881 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@78c8ec7e
   [junit4]   2> 188092 T881 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/shard-1-001/cores/collection1/data
   [junit4]   2> 188092 T881 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 188092 T881 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 188093 T881 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F22FA1F94A30AA2D-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 188094 T881 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=29, maxMergeAtOnceExplicit=33, maxMergedSegmentMB=86.779296875, floorSegmentMB=0.7470703125, forceMergeDeletesPctAllowed=22.320311402449484, segmentsPerTier=19.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 188095 T881 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@14dbcf58 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6b09a51a),segFN=segments_1,generation=1}
   [junit4]   2> 188095 T881 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 188102 T881 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 188103 T881 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 188103 T881 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 188104 T881 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 188104 T881 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 188104 T881 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 188105 T881 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 188105 T881 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 188106 T881 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 188133 T881 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 188135 T881 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 188138 T881 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 188140 T881 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 188148 T881 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/luke,/admin/file,/replication,/update,/admin/logging,standard,/admin/segments,/admin/plugins,/admin/ping,/admin/properties,/update/json/docs,/admin/system,/schema,/config,/admin/threads,/get,/update/csv,/update/json,/admin/mbeans
   [junit4]   2> 188151 T881 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 188153 T881 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 188153 T881 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 188155 T881 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 188155 T881 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 188156 T881 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=48, maxMergeAtOnceExplicit=20, maxMergedSegmentMB=72.1044921875, floorSegmentMB=1.9287109375, forceMergeDeletesPctAllowed=10.126703683436897, segmentsPerTier=15.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 188157 T881 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@14dbcf58 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6b09a51a),segFN=segments_1,generation=1}
   [junit4]   2> 188158 T881 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 188158 T881 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@2c7fc932[collection1] main
   [junit4]   2> 188158 T881 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 188159 T881 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 188160 T881 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 188160 T881 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 188160 T881 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 188161 T881 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 188161 T881 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 188162 T881 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 188162 T881 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 188162 T881 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 188164 T882 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2c7fc932[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 188164 T881 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 188165 T885 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:41958/iiknr collection:collection1 shard:shard1
   [junit4]   2> 188166 T824 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 188167 T824 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 188167 T885 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard

[...truncated too long message...]

/127.0.0.1:32585/
   [junit4]   2> 1644824 T7378 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:32573/
   [junit4]   2> 1644825 T7378 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 9 connections to: http://127.0.0.1:32564/
   [junit4]   2> 1644825 T7378 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:32600/
   [junit4]   2> 1644825 T7378 C:control_collection S:shard1 c:collection1 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 1644826 T7378 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:32585/
   [junit4]   2> 1644826 T7378 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:32573/
   [junit4]   2> 1644826 T7378 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:32564/
   [junit4]   2> 1644827 T7378 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:32600/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=F22FA1F94A30AA2D -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=fr_CH -Dtests.timezone=America/Argentina/San_Luis -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   76.8s J3 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([F22FA1F94A30AA2D:7A7B9E23E4CCC7D5]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 1644843 T7378 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest F22FA1F94A30AA2D-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {multiDefault=PostingsFormat(name=MockRandom), a_t=PostingsFormat(name=LuceneFixedGap), text=PostingsFormat(name=LuceneVarGapFixedInterval), range_facet_l=PostingsFormat(name=LuceneFixedGap), id=Lucene50(blocksize=128), timestamp=PostingsFormat(name=LuceneFixedGap), intDefault=PostingsFormat(name=LuceneFixedGap), _version_=PostingsFormat(name=LuceneFixedGap)}, docValues:{timestamp=DocValuesFormat(name=Lucene50)}, sim=RandomSimilarityProvider(queryNorm=true,coord=no): {}, locale=fr_CH, timezone=America/Argentina/San_Luis
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=225486360,total=383254528
   [junit4]   2> NOTE: All tests run in this JVM: [TestReloadAndDeleteDocs, TestReplicaProperties, TestDefaultStatsCache, TestFastOutputStream, TestManagedSynonymFilterFactory, TestStressUserVersions, TestStressLucene, ExitableDirectoryReaderTest, SuggesterFSTTest, TestRebalanceLeaders, TestBinaryField, TestSolrQueryParserDefaultOperatorResource, TestCoreDiscovery, MigrateRouteKeyTest, TestManagedSchemaDynamicFieldResource, TestBulkSchemaConcurrent, UpdateRequestProcessorFactoryTest, UniqFieldsUpdateProcessorFactoryTest, OverseerRolesTest, TestDynamicLoading, TermVectorComponentTest, StandardRequestHandlerTest, TestCSVLoader, TestPerFieldSimilarity, TestLeaderElectionZkExpiry, MergeStrategyTest, TestCloudManagedSchemaConcurrent, UpdateParamsTest, FullSolrCloudDistribCmdsTest, TestClassNameShortening, SharedFSAutoReplicaFailoverTest, SimpleCollectionCreateDeleteTest, DirectSolrSpellCheckerTest, TestDistributedGrouping, IndexSchemaRuntimeFieldTest, SynonymTokenizerTest, SignatureUpdateProcessorFactoryTest, OutOfBoxZkACLAndCredentialsProvidersTest, TestSolrQueryParserResource, LeaderElectionTest, TestCoreContainer, TestRecoveryHdfs, TestPivotHelperCode, TestAddFieldRealTimeGet, HdfsSyncSliceTest, TestHighlightDedupGrouping, TestStandardQParsers, TestFuzzyAnalyzedSuggestions, TestPartialUpdateDeduplication, TestElisionMultitermQuery, OverseerTest, SortByFunctionTest, TestChildDocTransformer, DateMathParserTest, ZkSolrClientTest, CursorPagingTest, BufferStoreTest, TestHdfsUpdateLog, TestMiniSolrCloudCluster, SyncSliceTest, ShardSplitTest, ReplicationFactorTest]
   [junit4] Completed on J3 in 76.87s, 1 test, 1 error <<< FAILURES!

[...truncated 729 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 1 failure, 62 ignored (22 assumptions)

Total time: 53 minutes 36 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 219089863 bytes
Compression is 0.0%
Took 41 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2850 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2850/

4 tests failed.
REGRESSION:  org.apache.solr.handler.component.DistributedMLTComponentTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:17455/fe_oqh/pr/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:17455/fe_oqh/pr/collection1
	at __randomizedtesting.SeedInfo.seed([2B6147B86513E014:A3357862CBEF8DEC]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:566)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:943)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:558)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:606)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:588)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:567)
	at org.apache.solr.handler.component.DistributedMLTComponentTest.test(DistributedMLTComponentTest.java:126)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 51 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:45557/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:45557/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([2B6147B86513E014:A3357862CBEF8DEC]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([2B6147B86513E014:A3357862CBEF8DEC]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([2B6147B86513E014:A3357862CBEF8DEC]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9801 lines...]
   [junit4] Suite: org.apache.solr.handler.component.DistributedMLTComponentTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/init-core-data-001
   [junit4]   2> 821614 T1534 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (true)
   [junit4]   2> 821614 T1534 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /fe_oqh/pr
   [junit4]   2> 823393 T1534 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/control/cores/collection1
   [junit4]   2> 823396 T1534 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 823401 T1534 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:17441
   [junit4]   2> 823401 T1534 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/fe_oqh/pr, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/control/cores, hostPort=17441}
   [junit4]   2> 823402 T1534 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 823402 T1534 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/control/'
   [junit4]   2> 823450 T1534 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/control/solr.xml
   [junit4]   2> 823497 T1534 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/control/cores
   [junit4]   2> 823497 T1534 oasc.CoreContainer.<init> New CoreContainer 1997000886
   [junit4]   2> 823498 T1534 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/control/]
   [junit4]   2> 823498 T1534 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/control/lib
   [junit4]   2> 823499 T1534 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/control/lib).
   [junit4]   2> 823508 T1534 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 823509 T1534 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 823509 T1534 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 823509 T1534 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 823509 T1534 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 823510 T1534 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 823510 T1534 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 823510 T1534 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 823510 T1534 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 823510 T1534 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 823511 T1534 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 823511 T1534 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 823511 T1534 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 823512 T1534 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 823512 T1534 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 823513 T1534 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/control/cores
   [junit4]   2> 823515 T1534 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/control/cores/collection1/, transient=false, schema=schema.xml, loadOnStartup=true, name=collection1, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/control/cores/collection1, configSet=collection1, dataDir=data/}
   [junit4]   2> 823515 T1534 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/control/cores/collection1/
   [junit4]   2> 823516 T1534 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 823517 T1544 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 823518 T1544 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 823518 T1544 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 823519 T1544 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 823563 T1544 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 823588 T1544 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 823645 T1544 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 823646 T1544 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 823654 T1544 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 823957 T1544 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 823970 T1544 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 823972 T1544 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 823980 T1544 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 823984 T1544 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 823986 T1544 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 823987 T1544 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 823987 T1544 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 823987 T1544 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 823988 T1544 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 823988 T1544 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 823988 T1544 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from configset /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1
   [junit4]   2> 823988 T1544 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 823989 T1544 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/control/cores/collection1/data/
   [junit4]   2> 823989 T1544 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2e2acf91
   [junit4]   2> 823990 T1544 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/control/cores/collection1/data
   [junit4]   2> 823990 T1544 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/control/cores/collection1/data/index/
   [junit4]   2> 823990 T1544 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/control/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 823990 T1544 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/control/cores/collection1/data/index
   [junit4]   2> 823990 T1544 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=18, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 823991 T1544 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3bf2799d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2c0e512a),segFN=segments_1,generation=1}
   [junit4]   2> 823991 T1544 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 823996 T1544 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 823997 T1544 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 823997 T1544 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 823997 T1544 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 823997 T1544 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 823997 T1544 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 823998 T1544 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 823998 T1544 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 823998 T1544 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 823998 T1544 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 823998 T1544 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 823999 T1544 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 824014 T1544 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 824016 T1544 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 824018 T1544 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 824019 T1544 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 824024 T1544 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 824027 T1544 oasc.RequestHandlers.initHandlersFromConfig Registered paths: tvrh,/config,/search-facet-invariants,mltrh,/mlt,defaults,/update/json/docs,/update/json,dismax,/debug/dump,/admin/mbeans,/update/csv,/admin/logging,spellCheckCompRH,/admin/threads,/admin/luke,spellCheckCompRH_Direct,/admin/segments,/admin/properties,/search-facet-def,/terms,lazy,/replication,mock,spellCheckCompRH1,/admin/ping,/get,standard,spellCheckWithWordbreak,/schema,/admin/system,/admin/plugins,spellCheckWithWordbreak_Direct,/update,/admin/file
   [junit4]   2> 824028 T1544 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 824029 T1544 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 824030 T1544 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 824031 T1544 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 824032 T1544 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 824032 T1544 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 824032 T1544 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=37, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 824033 T1544 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3bf2799d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2c0e512a),segFN=segments_1,generation=1}
   [junit4]   2> 824033 T1544 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 824033 T1544 oass.SolrIndexSearcher.<init> Opening Searcher@31557774[collection1] main
   [junit4]   2> 824034 T1544 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 824034 T1544 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 824034 T1544 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 824034 T1544 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 824042 T1544 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 824113 T1544 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 824113 T1545 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 824114 T1545 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 824114 T1545 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 824114 T1545 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 824114 T1544 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 824115 T1534 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 824115 T1534 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 824115 T1545 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 824116 T1545 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 824117 T1545 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 824117 T1545 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 824118 T1545 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 824119 T1545 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@31557774[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 826107 T1534 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard0/cores/collection1
   [junit4]   2> 826111 T1534 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 826114 T1534 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:17444
   [junit4]   2> 826114 T1534 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=17444, hostContext=/fe_oqh/pr, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard0/cores}
   [junit4]   2> 826115 T1534 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 826115 T1534 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard0/'
   [junit4]   2> 826179 T1534 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard0/solr.xml
   [junit4]   2> 826239 T1534 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard0/cores
   [junit4]   2> 826239 T1534 oasc.CoreContainer.<init> New CoreContainer 1726768816
   [junit4]   2> 826240 T1534 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard0/]
   [junit4]   2> 826240 T1534 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard0/lib
   [junit4]   2> 826241 T1534 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard0/lib).
   [junit4]   2> 826255 T1534 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 826256 T1534 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 826256 T1534 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 826256 T1534 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 826257 T1534 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 826257 T1534 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 826257 T1534 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 826257 T1534 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 826258 T1534 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 826258 T1534 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 826258 T1534 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 826259 T1534 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 826260 T1534 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 826260 T1534 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 826260 T1534 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 826261 T1534 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard0/cores
   [junit4]   2> 826264 T1534 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, dataDir=data/, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard0/cores/collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard0/cores/collection1/, configSet=collection1, schema=schema.xml, loadOnStartup=true, name=collection1}
   [junit4]   2> 826264 T1534 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard0/cores/collection1/
   [junit4]   2> 826265 T1534 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 826266 T1556 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 826268 T1556 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 826268 T1556 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 826269 T1556 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 826320 T1556 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 826352 T1556 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 826422 T1556 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 826423 T1556 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 826434 T1556 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 826788 T1556 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 826800 T1556 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 826802 T1556 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 826810 T1556 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 826813 T1556 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 826816 T1556 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 826817 T1556 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 826817 T1556 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 826817 T1556 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 826817 T1556 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 826818 T1556 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 826818 T1556 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from configset /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1
   [junit4]   2> 826818 T1556 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 826818 T1556 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard0/cores/collection1/data/
   [junit4]   2> 826819 T1556 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2e2acf91
   [junit4]   2> 826819 T1556 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard0/cores/collection1/data
   [junit4]   2> 826820 T1556 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard0/cores/collection1/data/index/
   [junit4]   2> 826820 T1556 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard0/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 826820 T1556 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard0/cores/collection1/data/index
   [junit4]   2> 826820 T1556 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=18, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 826821 T1556 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@41441464 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5c96d2b5),segFN=segments_1,generation=1}
   [junit4]   2> 826821 T1556 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 826828 T1556 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 826828 T1556 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 826828 T1556 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 826828 T1556 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 826829 T1556 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 826829 T1556 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 826829 T1556 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 826829 T1556 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 826829 T1556 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 826830 T1556 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 826830 T1556 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 826830 T1556 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 826843 T1556 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 826845 T1556 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 826846 T1556 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 826848 T1556 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 826855 T1556 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 826858 T1556 oasc.RequestHandlers.initHandlersFromConfig Registered paths: tvrh,/config,/search-facet-invariants,mltrh,/mlt,defaults,/update/json/docs,/update/json,dismax,/debug/dump,/admin/mbeans,/update/csv,/admin/logging,spellCheckCompRH,/admin/threads,/admin/luke,spellCheckCompRH_Direct,/admin/segments,/admin/properties,/search-facet-def,/terms,lazy,/replication,mock,spellCheckCompRH1,/admin/ping,/get,standard,spellCheckWithWordbreak,/schema,/admin/system,/admin/plugins,spellCheckWithWordbreak_Direct,/update,/admin/file
   [junit4]   2> 826860 T1556 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 826861 T1556 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 826862 T1556 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 826862 T1556 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 826863 T1556 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 826863 T1556 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 826864 T1556 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=37, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 826865 T1556 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@41441464 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5c96d2b5),segFN=segments_1,generation=1}
   [junit4]   2> 826865 T1556 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 826865 T1556 oass.SolrIndexSearcher.<init> Opening Searcher@6864cdda[collection1] main
   [junit4]   2> 826865 T1556 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 826866 T1556 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 826866 T1556 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 826866 T1556 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 826873 T1556 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 826922 T1556 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 826923 T1557 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 826924 T1557 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 826924 T1557 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 826924 T1557 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 826924 T1556 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 826925 T1534 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 826925 T1557 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 826925 T1534 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 826926 T1557 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 826927 T1557 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 826928 T1557 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 826929 T1557 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 826931 T1557 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6864cdda[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 827916 T1534 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard1/cores/collection1
   [junit4]   2> 827918 T1534 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 827921 T1534 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:17453
   [junit4]   2> 827921 T1534 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/fe_oqh/pr, hostPort=17453, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard1/cores}
   [junit4]   2> 827922 T1534 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 827922 T1534 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard1/'
   [junit4]   2> 827952 T1534 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard1/solr.xml
   [junit4]   2> 827988 T1534 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard1/cores
   [junit4]   2> 827988 T1534 oasc.CoreContainer.<init> New CoreContainer 722900325
   [junit4]   2> 827988 T1534 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard1/]
   [junit4]   2> 827989 T1534 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard1/lib
   [junit4]   2> 827989 T1534 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard1/lib).
   [junit4]   2> 827998 T1534 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 827999 T1534 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 827999 T1534 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 827999 T1534 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 827999 T1534 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 827999 T1534 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 828000 T1534 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 828000 T1534 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 828000 T1534 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 828000 T1534 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 828001 T1534 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 828001 T1534 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 828001 T1534 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 828002 T1534 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 828002 T1534 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 828003 T1534 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard1/cores
   [junit4]   2> 828004 T1534 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {dataDir=data/, configSet=collection1, config=solrconfig.xml, transient=false, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard1/cores/collection1/, name=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard1/cores/collection1, schema=schema.xml, loadOnStartup=true}
   [junit4]   2> 828005 T1534 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard1/cores/collection1/
   [junit4]   2> 828005 T1534 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 828006 T1568 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 828008 T1568 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 828008 T1568 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 828009 T1568 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 828058 T1568 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 828088 T1568 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 828169 T1568 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 828170 T1568 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 828181 T1568 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 828580 T1568 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 828597 T1568 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 828600 T1568 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 828612 T1568 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 828617 T1568 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 828622 T1568 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 828623 T1568 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 828623 T1568 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 828623 T1568 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 828624 T1568 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 828624 T1568 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 828624 T1568 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from configset /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1
   [junit4]   2> 828625 T1568 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 828625 T1568 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard1/cores/collection1/data/
   [junit4]   2> 828625 T1568 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2e2acf91
   [junit4]   2> 828626 T1568 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard1/cores/collection1/data
   [junit4]   2> 828627 T1568 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard1/cores/collection1/data/index/
   [junit4]   2> 828627 T1568 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard1/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 828627 T1568 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard1/cores/collection1/data/index
   [junit4]   2> 828628 T1568 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=18, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 828629 T1568 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2377e8c5 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1552efb4),segFN=segments_1,generation=1}
   [junit4]   2> 828629 T1568 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 828639 T1568 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 828639 T1568 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 828640 T1568 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 828640 T1568 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 828640 T1568 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 828640 T1568 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 828641 T1568 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 828641 T1568 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 828641 T1568 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 828642 T1568 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 828642 T1568 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 828642 T1568 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 828665 T1568 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 828668 T1568 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 828671 T1568 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 828673 T1568 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 828683 T1568 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 828689 T1568 oasc.RequestHandlers.initHandlersFromConfig Registered paths: tvrh,/config,/search-facet-invariants,mltrh,/mlt,defaults,/update/json/docs,/update/json,dismax,/debug/dump,/admin/mbeans,/update/csv,/admin/logging,spellCheckCompRH,/admin/threads,/admin/luke,spellCheckCompRH_Direct,/admin/segments,/admin/properties,/search-facet-def,/terms,lazy,/replication,mock,spellCheckCompRH1,/admin/ping,/get,standard,spellCheckWithWordbreak,/schema,/admin/system,/admin/plugins,spellCheckWithWordbreak_Direct,/update,/admin/file
   [junit4]   2> 828691 T1568 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 828692 T1568 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 828694 T1568 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 828694 T1568 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 828696 T1568 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 828696 T1568 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 828697 T1568 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=37, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 828698 T1568 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2377e8c5 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1552efb4),segFN=segments_1,generation=1}
   [junit4]   2> 828698 T1568 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 828699 T1568 oass.SolrIndexSearcher.<init> Opening Searcher@7cf3618f[collection1] main
   [junit4]   2> 828699 T1568 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 828699 T1568 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 828700 T1568 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 828700 T1568 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 828749 T1568 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 828802 T1568 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 828803 T1569 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 828804 T1569 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 828804 T1569 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 828804 T1569 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 828805 T1569 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 828805 T1568 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 828806 T1569 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 828806 T1534 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 828806 T1534 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 828806 T1569 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 828807 T1569 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 828808 T1569 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 828809 T1569 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7cf3618f[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 830154 T1534 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard2/cores/collection1
   [junit4]   2> 830156 T1534 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 830159 T1534 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:17455
   [junit4]   2> 830159 T1534 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard2/cores, hostPort=17455, hostContext=/fe_oqh/pr}
   [junit4]   2> 830160 T1534 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 830160 T1534 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard2/'
   [junit4]   2> 830202 T1534 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard2/solr.xml
   [junit4]   2> 830255 T1534 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard2/cores
   [junit4]   2> 830255 T1534 oasc.CoreContainer.<init> New CoreContainer 372587542
   [junit4]   2> 830255 T1534 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard2/]
   [junit4]   2> 830256 T1534 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard2/lib
   [junit4]   2> 830256 T1534 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard2/lib).
   [junit4]   2> 830266 T1534 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 830266 T1534 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 830267 T1534 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 830267 T1534 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 830267 T1534 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 830267 T1534 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 830268 T1534 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 830268 T1534 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 830268 T1534 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 830268 T1534 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 830268 T1534 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 830269 T1534 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 830269 T1534 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 830269 T1534 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 830270 T1534 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 830271 T1534 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard2/cores
   [junit4]   2> 830273 T1534 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {configSet=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedMLTComponentTest 2B6147B86513E014-001/tempDir-001/shard2/cores/collection1/, transient=false, dataDir=data/, config=solrconfig.xml, loadOnStartup=true, instanceDir=/usr/home/jenkins

[...truncated too long message...]

ed o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> 1638904 T3219 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:47085 47085
   [junit4]   2> 1638936 T3220 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:47085 47085
   [junit4]   2> 1638937 T3220 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1638937 T3219 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:37487/
   [junit4]   2> 1638938 T3219 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:13846/
   [junit4]   2> 1638938 T3219 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:15830/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=2B6147B86513E014 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=th -Dtests.timezone=Australia/Darwin -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   67.1s J1 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([2B6147B86513E014:A3357862CBEF8DEC]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 1638957 T3219 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 2B6147B86513E014-001
   [junit4]   2> 67063 T3218 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=HighCompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=HIGH_COMPRESSION, chunkSize=3, maxDocsPerChunk=89, blockSize=316), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=HIGH_COMPRESSION, chunkSize=3, blockSize=316)), sim=DefaultSimilarity, locale=th, timezone=Australia/Darwin
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=181191552,total=344981504
   [junit4]   2> NOTE: All tests run in this JVM: [XmlUpdateRequestHandlerTest, HdfsDirectoryTest, DocValuesMissingTest, HdfsWriteToMultipleCollectionsTest, TestPerFieldSimilarity, TestAnalyzeInfixSuggestions, TestManagedSynonymFilterFactory, SharedFSAutoReplicaFailoverTest, HdfsLockFactoryTest, TestBinaryField, BlockDirectoryTest, FastVectorHighlighterTest, TestBM25SimilarityFactory, SuggestComponentTest, HdfsBasicDistributedZkTest, TestStressVersions, TestBinaryResponseWriter, UpdateParamsTest, SpatialFilterTest, TestReload, TestManagedResource, TriLevelCompositeIdRoutingTest, DocValuesTest, SuggesterFSTTest, TestSchemaVersionResource, AutoCommitTest, TestMiniSolrCloudClusterSSL, TestLRUCache, ConnectionManagerTest, TestTolerantSearch, SOLR749Test, TestSearchPerf, LeaderFailoverAfterPartitionTest, TestCSVResponseWriter, InfoHandlerTest, TestFieldSortValues, TestRebalanceLeaders, LukeRequestHandlerTest, TestNamedUpdateProcessors, CoreAdminCreateDiscoverTest, TestBulkSchemaAPI, DirectUpdateHandlerOptimizeTest, TimeZoneUtilsTest, BufferStoreTest, TestInitParams, SolrCoreCheckLockOnStartupTest, AliasIntegrationTest, SpellCheckCollatorTest, SSLMigrationTest, DistributedMLTComponentTest, TestImplicitCoreProperties, TestMaxScoreQueryParser, RegexBoostProcessorTest, FieldAnalysisRequestHandlerTest, RequestHandlersTest, DeleteInactiveReplicaTest, TestManagedSchema, TestFunctionQuery, TestDistributedSearch, CollectionsAPIDistributedZkTest, FullSolrCloudDistribCmdsTest, TestCloudManagedSchema, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J1 in 67.59s, 1 test, 1 error <<< FAILURES!

[...truncated 697 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 4 errors, 62 ignored (22 assumptions)

Total time: 54 minutes 42 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 205661087 bytes
Compression is 0.0%
Took 37 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2849 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2849/

4 tests failed.
REGRESSION:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test

Error Message:
There were too many update fails (36 > 20) - we expect it can happen, but shouldn't easily

Stack Trace:
java.lang.AssertionError: There were too many update fails (36 > 20) - we expect it can happen, but shouldn't easily
	at __randomizedtesting.SeedInfo.seed([D4102244464B20CF:5C441D9EE8B74D37]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertFalse(Assert.java:68)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:230)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:26693/aa/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:26693/aa/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([D4102244464B20CF:5C441D9EE8B74D37]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([D4102244464B20CF:5C441D9EE8B74D37]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([D4102244464B20CF:5C441D9EE8B74D37]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9514 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/init-core-data-001
   [junit4]   2> 250138 T862 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /aa/
   [junit4]   2> 250144 T862 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 250145 T863 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 250245 T862 oasc.ZkTestServer.run start zk server on port:26670
   [junit4]   2> 250246 T862 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 250247 T862 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 250250 T870 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2623ce5e name:ZooKeeperConnection Watcher:127.0.0.1:26670 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 250250 T862 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 250251 T862 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 250251 T862 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 250253 T862 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 250254 T862 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 250255 T873 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4499bc1f name:ZooKeeperConnection Watcher:127.0.0.1:26670/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 250256 T862 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 250256 T862 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 250256 T862 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 250258 T862 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 250259 T862 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 250260 T862 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 250262 T862 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 250262 T862 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 250265 T862 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 250265 T862 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 250267 T862 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 250268 T862 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 250269 T862 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 250270 T862 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 250272 T862 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 250272 T862 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 250274 T862 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 250275 T862 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 250276 T862 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 250277 T862 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 250279 T862 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 250279 T862 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 250281 T862 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 250281 T862 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 250283 T862 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 250283 T862 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 250285 T862 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 250285 T862 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 251474 T862 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/control-001/cores/collection1
   [junit4]   2> 251478 T862 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 251482 T862 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:?????
   [junit4]   2> 251482 T862 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/aa, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/control-001/cores, hostPort=26673, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/tempDir-001/control/data}
   [junit4]   2> 251483 T862 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@5fb271f1
   [junit4]   2> 251483 T862 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/control-001/'
   [junit4]   2> 251522 T862 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/control-001/solr.xml
   [junit4]   2> 251565 T862 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/control-001/cores
   [junit4]   2> 251565 T862 oasc.CoreContainer.<init> New CoreContainer 1059255326
   [junit4]   2> 251565 T862 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/control-001/]
   [junit4]   2> 251566 T862 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/control-001/lib
   [junit4]   2> 251566 T862 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/control-001/lib).
   [junit4]   2> 251576 T862 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 251576 T862 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 251577 T862 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 251577 T862 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 251577 T862 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 251577 T862 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 251578 T862 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 251578 T862 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 251578 T862 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 251578 T862 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 251579 T862 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 251579 T862 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 251580 T862 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 251580 T862 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 251580 T862 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 251580 T862 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:26670/solr
   [junit4]   2> 251581 T862 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 251581 T862 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 251582 T862 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 251583 T885 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@726375bb name:ZooKeeperConnection Watcher:127.0.0.1:26670 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 251584 T862 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 251584 T862 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 251586 T862 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 251587 T888 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3e74ba60 name:ZooKeeperConnection Watcher:127.0.0.1:26670/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 251587 T862 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 251588 T862 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 251590 T862 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 251592 T862 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 251593 T862 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 251595 T862 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 251596 T862 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 251597 T862 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 251598 T862 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 251599 T862 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:26673_aa
   [junit4]   2> 251600 T862 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:26673_aa
   [junit4]   2> 251601 T862 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 251602 T862 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 251603 T862 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 251604 T862 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:26673_aa
   [junit4]   2> 251604 T862 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 251606 T862 oasc.Overseer.start Overseer (id=93555601651007491-127.0.0.1:26673_aa-n_0000000000) starting
   [junit4]   2> 251607 T862 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 251611 T862 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 251612 T890 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 251612 T862 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 251612 T889 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 251616 T862 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/control-001/cores
   [junit4]   2> 251618 T862 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {shard=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/control-001/cores/collection1/, dataDir=data/, transient=false, name=collection1, coreNodeName=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/control-001/cores/collection1, config=solrconfig.xml, schema=schema.xml, loadOnStartup=true, collection=control_collection}
   [junit4]   2> 251619 T862 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/control-001/cores/collection1/
   [junit4]   2> 251619 T862 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 251620 T892 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 251620 T892 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 251621 T892 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 251621 T888 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 251622 T889 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:26673_aa",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:26673/aa",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null} current state version: 0
   [junit4]   2> 251622 T889 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:26673_aa",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:26673/aa",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 251623 T889 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 251623 T889 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 251624 T888 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 252621 T892 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 252622 T892 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 252622 T892 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 252623 T892 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 252623 T892 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 252623 T892 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/control-001/cores/collection1/'
   [junit4]   2> 252647 T892 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 252650 T892 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 252661 T892 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 252678 T892 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 252716 T892 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 252717 T892 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 252726 T892 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 253050 T892 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 253067 T892 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 253069 T892 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 253080 T892 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 253084 T892 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 253088 T892 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 253089 T892 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 253089 T892 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 253089 T892 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 253090 T892 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 253090 T892 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 253090 T892 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 253091 T892 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 253091 T892 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/control-001/cores/collection1/data/
   [junit4]   2> 253091 T892 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7c299e34
   [junit4]   2> 253092 T892 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/control-001/cores/collection1/data
   [junit4]   2> 253092 T892 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/control-001/cores/collection1/data/index/
   [junit4]   2> 253093 T892 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 253093 T892 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/control-001/cores/collection1/data/index
   [junit4]   2> 253093 T892 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=459365519, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 253094 T892 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1155623c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7ca45a07),segFN=segments_1,generation=1}
   [junit4]   2> 253094 T892 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 253100 T892 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 253100 T892 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 253101 T892 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 253101 T892 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 253101 T892 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 253101 T892 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 253102 T892 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 253102 T892 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 253102 T892 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 253124 T892 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 253127 T892 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 253130 T892 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 253132 T892 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 253139 T892 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/file,/admin/threads,/config,/replication,/schema,/update/csv,/admin/logging,/admin/ping,/get,/admin/mbeans,/admin/segments,/admin/plugins,standard,/admin/system,/update,/admin/properties,/admin/luke,/update/json/docs,/update/json
   [junit4]   2> 253142 T892 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 253143 T892 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 253144 T892 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 253146 T892 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 253146 T892 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 253147 T892 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=28, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.20197069233172302]
   [junit4]   2> 253148 T892 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1155623c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7ca45a07),segFN=segments_1,generation=1}
   [junit4]   2> 253148 T892 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 253149 T892 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@6777f27a[collection1] main
   [junit4]   2> 253149 T892 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 253150 T892 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 253150 T892 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 253151 T892 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 253151 T892 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 253152 T892 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 253152 T892 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 253153 T892 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 253153 T892 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 253153 T892 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 253154 T893 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6777f27a[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 253155 T892 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 253156 T896 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:26673/aa collection:control_collection shard:shard1
   [junit4]   2> 253156 T862 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 253157 T862 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 253158 T896 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 253162 T862 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 253163 T896 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 253164 T862 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 253165 T900 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@50fa1a9c name:ZooKeeperConnection Watcher:127.0.0.1:26670/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 253166 T862 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 253166 T896 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 253166 T888 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 253167 T862 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 253167 T896 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 253168 T889 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 253167 T862 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> ASYNC  NEW_CORE C54 name=collection1 org.apache.solr.core.SolrCore@1e3d7730 url=http://127.0.0.1:26673/aa/collection1 node=127.0.0.1:26673_aa C54_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:26673_aa, base_url=http://127.0.0.1:26673/aa, state=down, core=collection1}
   [junit4]   2> 253168 T896 C:control_collection S:shard1 C54 P26673 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:26673/aa/collection1/
   [junit4]   2> 253169 T896 C:control_collection S:shard1 C54 P26673 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 253170 T896 C:control_collection S:shard1 C54 P26673 oasc.SyncStrategy.syncToMe http://127.0.0.1:26673/aa/collection1/ has no replicas
   [junit4]   2> 253170 T888 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 253170 T900 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 253170 T896 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:26673/aa/collection1/ shard1
   [junit4]   2> 253172 T896 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 253172 T862 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 253177 T888 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 253178 T889 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:26673/aa",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 253282 T888 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 253283 T900 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 253330 T896 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:26673/aa/collection1/ and leader is http://127.0.0.1:26673/aa/collection1/
   [junit4]   2> 253331 T896 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:26673/aa
   [junit4]   2> 253331 T896 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 253332 T896 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 253332 T896 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 253334 T888 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 253335 T889 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:26673_aa",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:26673/aa",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1"} current state version: 3
   [junit4]   2> 253336 T889 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:26673_aa",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:26673/aa",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1"}
   [junit4]   2> 253440 T900 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 253441 T888 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 255108 T862 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/shard-1-001/cores/collection1
   [junit4]   2> 255112 T862 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/shard-1-001
   [junit4]   2> 255113 T862 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 255115 T862 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:?????
   [junit4]   2> 255116 T862 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/aa, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/tempDir-001/jetty1, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/shard-1-001/cores, solrconfig=solrconfig.xml, hostPort=26693}
   [junit4]   2> 255116 T862 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@5fb271f1
   [junit4]   2> 255116 T862 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/shard-1-001/'
   [junit4]   2> 255165 T862 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/shard-1-001/solr.xml
   [junit4]   2> 255224 T862 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/shard-1-001/cores
   [junit4]   2> 255224 T862 oasc.CoreContainer.<init> New CoreContainer 2077741704
   [junit4]   2> 255225 T862 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/shard-1-001/]
   [junit4]   2> 255225 T862 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/shard-1-001/lib
   [junit4]   2> 255226 T862 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/shard-1-001/lib).
   [junit4]   2> 255236 T862 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 255237 T862 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 255237 T862 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 255237 T862 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 255237 T862 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 255238 T862 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 255238 T862 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 255238 T862 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 255238 T862 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 255239 T862 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 255239 T862 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 255239 T862 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 255240 T862 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 255240 T862 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 255240 T862 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 255241 T862 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:26670/solr
   [junit4]   2> 255241 T862 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 255241 T862 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 255242 T862 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 255243 T912 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6f006ed8 name:ZooKeeperConnection Watcher:127.0.0.1:26670 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 255244 T862 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 255245 T862 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 255246 T862 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 255247 T915 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@42b0d33 name:ZooKeeperConnection Watcher:127.0.0.1:26670/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 255247 T862 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 255251 T862 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 256255 T862 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:26693_aa
   [junit4]   2> 256256 T862 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:26693_aa
   [junit4]   2> 256259 T862 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 256261 T862 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/shard-1-001/cores
   [junit4]   2> 256263 T862 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {dataDir=data/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/shard-1-001/cores/collection1, schema=schema.xml, collection=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/shard-1-001/cores/collection1/, name=collection1, coreNodeName=, transient=false, loadOnStartup=true, config=solrconfig.xml, shard=}
   [junit4]   2> 256263 T862 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/shard-1-001/cores/collection1/
   [junit4]   2> 256263 T862 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 256264 T916 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 256265 T916 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 256266 T916 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 256266 T888 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 256266 T889 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:26693_aa",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:26693/aa",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null} current state version: 4
   [junit4]   2> 256267 T889 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:26693_aa",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:26693/aa",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 256267 T889 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 256267 T889 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 256269 T915 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 256269 T900 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 256269 T888 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 257266 T916 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 257267 T916 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 257268 T916 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 257268 T916 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 257269 T916 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 257269 T916 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/shard-1-001/cores/collection1/'
   [junit4]   2> 257289 T916 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 257292 T916 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 257305 T916 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 257330 T916 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 257383 T916 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 257384 T916 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 257397 T916 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 257744 T916 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 257757 T916 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 257759 T916 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 257773 T916 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 257779 T916 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 257783 T916 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 257784 T916 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 257784 T916 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 257784 T916 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 257785 T916 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 257785 T916 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 257786 T916 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 257786 T916 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 257786 T916 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 257786 T916 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7c299e34
   [junit4]   2> 257787 T916 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/shard-1-001/cores/collection1/data
   [junit4]   2> 257788 T916 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 257788 T916 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 257788 T916 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D4102244464B20CF-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 257789 T916 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=459365519, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 257789 T916 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5f1ec010 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5bd38b39),segFN=segments_1,generation=1}
   [junit4]   2> 257790 T916 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 257796 T916 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 257796 T916 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 257797 T916 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 257797 T916 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 257797 T916 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 257797 T916 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 257798 T916 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 257798 T916 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 257798 T916 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 257812 T916 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 257814 T916 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 257816 T916 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 257818 T916 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 257825 T916 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/file,/admin/threads,/config,/replication,/schema,/update/csv,/admin/logging,/admin/ping,/get,/admin/mbeans,/admin/segments,/admin/plugins,standard,/admin/system,/update,/admin/properties,/admin/luke,/update/json/docs,/update/json
   [junit4]   2> 257826 T916 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 257828 T916 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 257828 T916 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 257829 T916 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 257829 T916 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 257830 T916 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=28, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.20197069233172302]
   [junit4]   2> 257831 T916 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5f1ec010 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5bd38b39),segFN=segments_1,generation=1}
   [junit4]   2> 257831 T916 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 257831 T916 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@57e343f2[collection1] main
   [junit4]   2> 257832 T916 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 257833 T916 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 257833 T916 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 257833 T916 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 257833 T916 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 257834 T916 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 257834 T916 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 257834 T916 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 257835 T916 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 257835 T916 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 257836 T917 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@57e343f2[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 257837 T916 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 257838 T920 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:26693/aa collection:collection1 shard:shard1
   [junit4]   2> 257838 T862 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 257839 T862 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 257839 T920 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 257843 T920 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 257845 T888 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 257845 T920 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 257846 T920 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C55 name=collection1 org.apache.solr.core.SolrCore@6b8e7a08 url=http://127.0.0.1:26693/aa/collection1 node=127.0.0.1:26693_aa C55_STATE=coll:collection1 core:collection1 props:{node_name=127.0.0.1:26693_aa, base_url=http://127.0.0.1:26693/aa, state=down, core=collection1}
   [junit4]   2> 257846 T920 C:collection1 S:shard1 C55 P26693 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:26693/aa/collection1/
   [junit4]   2> 257846 T889 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 257846 T920 C:collection1 S:shard1 C55 P26693 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 257847 T920 C:collection1 S:shard1 C55 P26693 oasc.SyncStrategy.syncToMe http://127.0.0.1:26693/aa/collection1/ has no replicas
   [junit4]   2> 257847 T920 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:26693/aa/collection1/ shard1
   [junit4]   2> 257848 T920 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1

[...truncated too long message...]

]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:328)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:325)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:61)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:325)
   [junit4]   2> 	at org.apache.solr.cloud.DistributedQueue.orderedChildren(DistributedQueue.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.DistributedQueue.remove(DistributedQueue.java:192)
   [junit4]   2> 	at org.apache.solr.cloud.DistributedQueue.poll(DistributedQueue.java:619)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:238)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> 1453283 T5909 oasc.Overseer$ClusterStateUpdater.run Overseer Loop exiting : 127.0.0.1:32399_
   [junit4]   2> 1454085 T5899 C2666 P32399 oasc.SyncStrategy.sync WARN Closed, skipping sync up.
   [junit4]   2> 1454086 T5899 oasc.ShardLeaderElectionContext.rejoinLeaderElection Not rejoining election because CoreContainer is closed
   [junit4]   2> 1454086 T5899 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@54e46cad
   [junit4]   2> 1454086 T5899 oasc.ZkController.unRegisterConfListener  a listener was removed because of core close
   [junit4]   2> 1454087 T5899 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=2,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=695,cumulative_deletesById=333,cumulative_deletesByQuery=1,cumulative_errors=0,transaction_logs_total_size=98204,transaction_logs_total_number=1}
   [junit4]   2> 1454088 T5899 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 1454088 T5899 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 1454088 T5899 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 1454089 T5899 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 1454099 T5899 oasc.CachingDirectoryFactory.close Closing StandardDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 1454100 T5899 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest D4102244464B20CF-001/shard-3-001/cores/collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest D4102244464B20CF-001/shard-3-001/cores/collection1/data;done=false>>]
   [junit4]   2> 1454100 T5899 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest D4102244464B20CF-001/shard-3-001/cores/collection1/data
   [junit4]   2> 1454100 T5899 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest D4102244464B20CF-001/shard-3-001/cores/collection1/data/index [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest D4102244464B20CF-001/shard-3-001/cores/collection1/data/index;done=false>>]
   [junit4]   2> 1454100 T5899 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest D4102244464B20CF-001/shard-3-001/cores/collection1/data/index
   [junit4]   2> 1454101 T5745 C:control_collection S:shard1 c:collection1 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> 1454304 T5745 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:50877 50877
   [junit4]   2> 1454409 T5873 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:50877 50877
   [junit4]   2> 1454410 T5873 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		2	/solr/overseer/queue
   [junit4]   2> 		2	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ChaosMonkeyNothingIsSafeTest -Dtests.method=test -Dtests.seed=D4102244464B20CF -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=bg_BG -Dtests.timezone=Atlantic/Faeroe -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE  279s J2 | ChaosMonkeyNothingIsSafeTest.test <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: There were too many update fails (36 > 20) - we expect it can happen, but shouldn't easily
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([D4102244464B20CF:5C441D9EE8B74D37]:0)
   [junit4]    > 	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:230)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1454419 T5745 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeyNothingIsSafeTest D4102244464B20CF-001
   [junit4]   2> 278588 T5744 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarityProvider(queryNorm=true,coord=crazy): {}, locale=bg_BG, timezone=Atlantic/Faeroe
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=50401872,total=412614656
   [junit4]   2> NOTE: All tests run in this JVM: [TestExtendedDismaxParser, FastVectorHighlighterTest, TestDefaultStatsCache, DocValuesTest, TestFieldTypeCollectionResource, SolrCoreTest, TestStressRecovery, TestPHPSerializedResponseWriter, SOLR749Test, TestCloudSchemaless, TestImplicitCoreProperties, DefaultValueUpdateProcessorTest, DocValuesMissingTest, HdfsDirectoryFactoryTest, TestReqParamsAPI, EchoParamsTest, DistributedSpellCheckComponentTest, TestSchemaNameResource, IndexBasedSpellCheckerTest, TestShortCircuitedRequests, TestStressLucene, TestRangeQuery, CircularListTest, TestSearchPerf, TestSchemaResource, TestManagedSchema, TestDistributedSearch, TestSortingResponseWriter, OutOfBoxZkACLAndCredentialsProvidersTest, TestOverriddenPrefixQueryForCustomFieldType, TestInitParams, FullSolrCloudDistribCmdsTest, AssignTest, TestExactStatsCache, TestSweetSpotSimilarityFactory, OutputWriterTest, TestUpdate, TestSolrCoreProperties, TestRemoteStreaming, TestBadConfig, TestValueSourceCache, DirectSolrSpellCheckerTest, SharedFSAutoReplicaFailoverTest, TestCloudManagedSchema, TestRandomFaceting, ZkCLITest, TimeZoneUtilsTest, DeleteReplicaTest, AutoCommitTest, AnalysisErrorHandlingTest, HdfsBasicDistributedZk2Test, QueryParsingTest, TestWordDelimiterFilterFactory, TestExpandComponent, TestComplexPhraseQParserPlugin, TestMiniSolrCloudClusterSSL, DistributedIntervalFacetingTest, BadCopyFieldTest, TestCloudManagedSchemaConcurrent, ChaosMonkeyNothingIsSafeTest]
   [junit4] Completed on J2 in 278.88s, 1 test, 1 failure <<< FAILURES!

[...truncated 730 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 1 failure, 62 ignored (22 assumptions)

Total time: 53 minutes 43 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 199727066 bytes
Compression is 0.0%
Took 39 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2848 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2848/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:39098/pofdg/e/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:39098/pofdg/e/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([4CA337679B49D543:C4F708BD35B5B8BB]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:62678/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:62678/collection1
	at __randomizedtesting.SeedInfo.seed([4CA337679B49D543:C4F708BD35B5B8BB]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:62678/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:36921/repfacttest_c8n_1x3_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:36921/repfacttest_c8n_1x3_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([4CA337679B49D543:C4F708BD35B5B8BB]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:36921/repfacttest_c8n_1x3_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9479 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/init-core-data-001
   [junit4]   2> 243193 T910 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /pofdg/e
   [junit4]   2> 243204 T910 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 243206 T911 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 243305 T910 oasc.ZkTestServer.run start zk server on port:39087
   [junit4]   2> 243306 T910 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 243308 T910 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 243316 T918 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3214093d name:ZooKeeperConnection Watcher:127.0.0.1:39087 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 243316 T910 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 243318 T910 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 243318 T910 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 243322 T910 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 243324 T910 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 243325 T921 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@beed7a0 name:ZooKeeperConnection Watcher:127.0.0.1:39087/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 243326 T910 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 243327 T910 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 243327 T910 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 243331 T910 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 243334 T910 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 243337 T910 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 243341 T910 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 243343 T910 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 243350 T910 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 243353 T910 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 243358 T910 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 243360 T910 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 243364 T910 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 243366 T910 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 243371 T910 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 243373 T910 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 243378 T910 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 243380 T910 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 243385 T910 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 243388 T910 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 243393 T910 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 243395 T910 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 243400 T910 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 243402 T910 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 243407 T910 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 243409 T910 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 243415 T910 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 243416 T910 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 243423 T912 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x14c5f7724630001, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> 247372 T910 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/control-001/cores/collection1
   [junit4]   2> 247383 T910 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 247390 T910 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:39099
   [junit4]   2> 247391 T910 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/pofdg/e, hostPort=39098, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/tempDir-001/control/data}
   [junit4]   2> 247393 T910 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 247393 T910 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/control-001/'
   [junit4]   2> 247497 T910 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/control-001/solr.xml
   [junit4]   2> 247606 T910 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/control-001/cores
   [junit4]   2> 247607 T910 oasc.CoreContainer.<init> New CoreContainer 525122361
   [junit4]   2> 247607 T910 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/control-001/]
   [junit4]   2> 247608 T910 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/control-001/lib
   [junit4]   2> 247611 T910 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/control-001/lib).
   [junit4]   2> 247654 T910 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 247654 T910 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 247655 T910 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 247655 T910 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 247656 T910 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 247656 T910 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 247657 T910 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 247658 T910 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 247658 T910 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 247659 T910 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 247659 T910 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 247660 T910 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 247661 T910 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 247662 T910 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 247662 T910 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 247663 T910 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:39087/solr
   [junit4]   2> 247663 T910 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 247664 T910 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 247665 T910 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 247668 T933 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3abe0d36 name:ZooKeeperConnection Watcher:127.0.0.1:39087 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 247669 T910 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 247671 T910 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 247674 T910 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 247676 T936 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5de4c159 name:ZooKeeperConnection Watcher:127.0.0.1:39087/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 247676 T910 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 247679 T910 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 247683 T910 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 247686 T910 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 247689 T910 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 247692 T910 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 247696 T910 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 247700 T910 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 247701 T910 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 247703 T910 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39098_pofdg%2Fe
   [junit4]   2> 247704 T910 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:39098_pofdg%2Fe
   [junit4]   2> 247707 T910 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 247710 T910 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 247712 T910 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 247715 T910 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:39098_pofdg%2Fe
   [junit4]   2> 247716 T910 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 247718 T910 oasc.Overseer.start Overseer (id=93554657584152579-127.0.0.1:39098_pofdg%2Fe-n_0000000000) starting
   [junit4]   2> 247722 T910 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 247732 T910 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 247733 T938 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 247733 T910 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 247735 T937 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 247741 T910 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/control-001/cores
   [junit4]   2> 247748 T910 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {coreNodeName=, shard=, name=collection1, transient=false, config=solrconfig.xml, dataDir=data/, schema=schema.xml, collection=control_collection, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/control-001/cores/collection1/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/control-001/cores/collection1, loadOnStartup=true}
   [junit4]   2> 247749 T910 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/control-001/cores/collection1/
   [junit4]   2> 247749 T910 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 247752 T940 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 247752 T940 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 247754 T936 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 247754 T940 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 247755 T937 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:39098/pofdg/e",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:39098_pofdg%2Fe",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null} current state version: 0
   [junit4]   2> 247757 T937 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:39098/pofdg/e",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:39098_pofdg%2Fe",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 247757 T937 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 247758 T937 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 247760 T936 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 248755 T940 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 248756 T940 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 248757 T940 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 248757 T940 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 248758 T940 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 248759 T940 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/control-001/cores/collection1/'
   [junit4]   2> 248782 T940 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 248786 T940 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 248812 T940 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 248854 T940 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 248929 T940 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 248931 T940 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 248950 T940 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 249600 T940 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 249631 T940 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 249636 T940 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 249657 T940 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 249662 T940 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 249667 T940 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 249668 T940 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 249668 T940 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 249669 T940 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 249670 T940 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 249670 T940 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 249670 T940 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 249671 T940 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 249671 T940 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/control-001/cores/collection1/data/
   [junit4]   2> 249671 T940 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@608ffa3
   [junit4]   2> 249673 T940 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/control-001/cores/collection1/data
   [junit4]   2> 249673 T940 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/control-001/cores/collection1/data/index/
   [junit4]   2> 249673 T940 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 249674 T940 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/control-001/cores/collection1/data/index
   [junit4]   2> 249676 T940 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=76401131, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 249677 T940 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5d4539e4 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6bdbd792),segFN=segments_1,generation=1}
   [junit4]   2> 249678 T940 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 249687 T940 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 249688 T940 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 249688 T940 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 249689 T940 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 249689 T940 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 249689 T940 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 249690 T940 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 249690 T940 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 249691 T940 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 249751 T940 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 249756 T940 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 249760 T940 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 249763 T940 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 249770 T940 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /get,/admin/system,/update/csv,standard,/admin/luke,/config,/admin/ping,/admin/properties,/admin/threads,/schema,/admin/plugins,/admin/segments,/replication,/admin/file,/update/json/docs,/update/json,/admin/mbeans,/update,/admin/logging
   [junit4]   2> 249773 T940 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 249775 T940 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 249775 T940 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 249778 T940 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 249778 T940 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 249779 T940 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=22, maxMergedSegmentMB=25.267578125, floorSegmentMB=0.8330078125, forceMergeDeletesPctAllowed=23.394277717059822, segmentsPerTier=16.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 249780 T940 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5d4539e4 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6bdbd792),segFN=segments_1,generation=1}
   [junit4]   2> 249780 T940 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 249781 T940 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@1aed04a9[collection1] main
   [junit4]   2> 249781 T940 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 249782 T940 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 249783 T940 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 249783 T940 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 249784 T940 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 249784 T940 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 249785 T940 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 249786 T940 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 249786 T940 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 249787 T940 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 249788 T941 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1aed04a9[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 249789 T940 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 249790 T944 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:39098/pofdg/e collection:control_collection shard:shard1
   [junit4]   2> 249792 T910 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 249792 T910 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 249792 T944 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 249798 T910 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 249800 T910 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 249800 T944 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 249802 T948 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3a6054b1 name:ZooKeeperConnection Watcher:127.0.0.1:39087/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 249803 T910 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 249803 T910 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 249803 T936 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 249804 T910 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 249804 T944 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 249805 T937 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 249805 T944 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C100 name=collection1 org.apache.solr.core.SolrCore@59d559f3 url=http://127.0.0.1:39098/pofdg/e/collection1 node=127.0.0.1:39098_pofdg%2Fe C100_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:39098/pofdg/e, state=down, node_name=127.0.0.1:39098_pofdg%2Fe, core=collection1}
   [junit4]   2> 249806 T944 C:control_collection S:shard1 C100 P39098 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:39098/pofdg/e/collection1/
   [junit4]   2> 249806 T944 C:control_collection S:shard1 C100 P39098 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 249806 T944 C:control_collection S:shard1 C100 P39098 oasc.SyncStrategy.syncToMe http://127.0.0.1:39098/pofdg/e/collection1/ has no replicas
   [junit4]   2> 249807 T944 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:39098/pofdg/e/collection1/ shard1
   [junit4]   2> 249807 T944 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 249807 T948 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 249807 T936 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 249808 T910 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 249813 T936 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 249815 T937 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:39098/pofdg/e",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 249919 T936 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 249919 T948 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 249966 T944 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:39098/pofdg/e/collection1/ and leader is http://127.0.0.1:39098/pofdg/e/collection1/
   [junit4]   2> 249967 T944 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:39098/pofdg/e
   [junit4]   2> 249967 T944 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 249968 T944 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 249968 T944 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 249971 T936 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 249972 T937 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:39098/pofdg/e",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:39098_pofdg%2Fe",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1"} current state version: 3
   [junit4]   2> 249973 T937 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:39098/pofdg/e",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:39098_pofdg%2Fe",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1"}
   [junit4]   2> 250078 T936 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 250078 T948 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 253274 T910 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/shard-1-001/cores/collection1
   [junit4]   2> 253278 T910 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/shard-1-001
   [junit4]   2> 253279 T910 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 253284 T910 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:39108
   [junit4]   2> 253285 T910 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/shard-1-001/cores, solrconfig=solrconfig.xml, hostPort=39107, hostContext=/pofdg/e, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/tempDir-001/jetty1}
   [junit4]   2> 253286 T910 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 253286 T910 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/shard-1-001/'
   [junit4]   2> 253410 T910 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/shard-1-001/solr.xml
   [junit4]   2> 253579 T910 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/shard-1-001/cores
   [junit4]   2> 253580 T910 oasc.CoreContainer.<init> New CoreContainer 619019723
   [junit4]   2> 253580 T910 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/shard-1-001/]
   [junit4]   2> 253581 T910 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/shard-1-001/lib
   [junit4]   2> 253583 T910 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/shard-1-001/lib).
   [junit4]   2> 253606 T910 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 253607 T910 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 253608 T910 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 253608 T910 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 253609 T910 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 253610 T910 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 253610 T910 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 253611 T910 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 253611 T910 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 253612 T910 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 253612 T910 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 253613 T910 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 253614 T910 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 253615 T910 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 253615 T910 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 253616 T910 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:39087/solr
   [junit4]   2> 253616 T910 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 253617 T910 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 253618 T910 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 253621 T960 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@555bffd2 name:ZooKeeperConnection Watcher:127.0.0.1:39087 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 253622 T910 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 253624 T910 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 253627 T910 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 253629 T963 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@469bf5a4 name:ZooKeeperConnection Watcher:127.0.0.1:39087/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 253630 T910 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 253637 T910 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 254643 T910 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39107_pofdg%2Fe
   [junit4]   2> 254645 T910 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:39107_pofdg%2Fe
   [junit4]   2> 254650 T910 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 254655 T910 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/shard-1-001/cores
   [junit4]   2> 254659 T910 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {dataDir=data/, loadOnStartup=true, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/shard-1-001/cores/collection1, schema=schema.xml, name=collection1, collection=collection1, shard=, coreNodeName=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/shard-1-001/cores/collection1/, config=solrconfig.xml}
   [junit4]   2> 254660 T910 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/shard-1-001/cores/collection1/
   [junit4]   2> 254661 T910 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 254664 T964 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 254664 T964 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 254666 T964 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 254666 T936 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 254668 T937 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:39107/pofdg/e",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:39107_pofdg%2Fe",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null} current state version: 4
   [junit4]   2> 254669 T937 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:39107/pofdg/e",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:39107_pofdg%2Fe",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 254669 T937 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 254670 T937 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 254672 T936 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 254673 T963 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 254672 T948 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 255667 T964 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 255668 T964 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 255670 T964 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 255670 T964 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 255672 T964 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 255672 T964 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/shard-1-001/cores/collection1/'
   [junit4]   2> 255700 T964 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 255705 T964 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 255729 T964 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 255756 T964 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 255823 T964 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 255824 T964 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 255851 T964 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 256513 T964 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 256544 T964 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 256548 T964 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 256573 T964 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 256578 T964 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 256583 T964 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 256584 T964 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 256584 T964 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 256585 T964 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 256585 T964 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 256586 T964 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 256586 T964 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 256586 T964 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 256587 T964 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 256587 T964 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@608ffa3
   [junit4]   2> 256588 T964 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/shard-1-001/cores/collection1/data
   [junit4]   2> 256589 T964 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 256589 T964 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 256589 T964 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 4CA337679B49D543-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 256591 T964 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=76401131, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 256592 T964 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6927e7c0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3ae0242f),segFN=segments_1,generation=1}
   [junit4]   2> 256592 T964 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 256601 T964 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 256602 T964 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 256602 T964 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 256602 T964 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 256603 T964 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 256603 T964 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 256603 T964 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 256604 T964 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 256604 T964 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 256636 T964 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 256642 T964 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 256647 T964 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 256653 T964 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 256667 T964 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /get,/admin/system,/update/csv,standard,/admin/luke,/config,/admin/ping,/admin/properties,/admin/threads,/schema,/admin/plugins,/admin/segments,/replication,/admin/file,/update/json/docs,/update/json,/admin/mbeans,/update,/admin/logging
   [junit4]   2> 256672 T964 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 256676 T964 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 256676 T964 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 256680 T964 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 256681 T964 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 256683 T964 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=22, maxMergedSegmentMB=25.267578125, floorSegmentMB=0.8330078125, forceMergeDeletesPctAllowed=23.394277717059822, segmentsPerTier=16.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 256684 T964 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6927e7c0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3ae0242f),segFN=segments_1,generation=1}
   [junit4]   2> 256685 T964 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 256685 T964 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@3dcb7efa[collection1] main
   [junit4]   2> 256686 T964 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 256687 T964 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 256687 T964 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 256687 T964 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 256688 T964 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 256688 T964 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 256689 T964 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 256689 T964 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 256689 T964 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 256689 T964 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 256691 T965 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3dcb7efa[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 256692 T964 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 256693 T968 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:39107/pofdg/e collection:collection1 shard:shard1
   [junit4]   2> 256694 T910 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 256695 T910 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 256695 T968 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 256702 T968 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 256705 T936 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 256705 T968 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 256706 T968 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C101 name=collection1 org.apache.solr.core.SolrCore@64a1e2dd url=http://127.0.0.1:39107/pofdg/e/collection1 node=127.0.0.1:39107_pofdg%2Fe C101_STATE=coll:collection1 core:collection1 props:{base_url=http://127.0.0.1:39107/pofdg/e, state=down, node_name=127.0.0.1:39107_pofdg%2Fe, core=collection1}
   [junit4]   2> 256706 T968 C:collection1 S:shard1 C101 P39107 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:39107/pofdg/e/collection1/
   [junit4]   2> 256706 T937 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 256707 T968 C:collection1 S:shard1 C101 P39107 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 256707 T968 C:collection1 S:shard1 C101 P39107 oasc.SyncStrategy.syncToMe http://127.0.0.1:39107/pofdg/e/collection1/ has no replicas
   [junit4]   2> 256708 T968 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:39107/pofdg/e/collection1/ shard1
   [junit4]   2> 256708 T968 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 256709 T948 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 256709 T936 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 256709 T963 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 256716 T936 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 256717 T937 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:39107/pofdg/e",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 256822 T936 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 256822 T963 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 256822 T948 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 256870 T968 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:39107/pofdg/e/collection1/ and leader is http://127.0.0.1:39107/pofdg/e/collection1/
   [junit4]   2> 256870 T968 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:39107/pofdg/e
   [junit4]   2> 256870 T968 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 256871 T968 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 256871 T968 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 256873 T936 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 256875 T937 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:39107/pofdg/e",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:39107_pofdg%2Fe",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard1"} current state version: 7
   [junit4]   2> 256876 T937 oasco.ReplicaMutator.updateState 

[...truncated too long message...]

 WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1607169 T6241 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:32069/
   [junit4]   2> 1607169 T6241 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:52950/
   [junit4]   2> 1607170 T6241 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:62678/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=4CA337679B49D543 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ar_SY -Dtests.timezone=Etc/GMT-4 -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   69.6s J3 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:62678/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([4CA337679B49D543:C4F708BD35B5B8BB]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:62678/collection1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 49 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 54 more
   [junit4]   2> 1607184 T6241 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4CA337679B49D543-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {_version_=BlockTreeOrds(blocksize=128), range_facet_l=BlockTreeOrds(blocksize=128), multiDefault=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), a_t=BlockTreeOrds(blocksize=128), timestamp=BlockTreeOrds(blocksize=128), intDefault=BlockTreeOrds(blocksize=128), text=PostingsFormat(name=Memory doPackFST= true), id=FSTOrd50}, docValues:{timestamp=DocValuesFormat(name=Memory)}, sim=RandomSimilarityProvider(queryNorm=false,coord=yes): {}, locale=ar_SY, timezone=Etc/GMT-4
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=129168888,total=400556032
   [junit4]   2> NOTE: All tests run in this JVM: [SampleTest, DistribJoinFromCollectionTest, ScriptEngineTest, IndexSchemaRuntimeFieldTest, TestDynamicFieldResource, TestMissingGroups, OverseerTest, HdfsChaosMonkeySafeLeaderTest, SimpleMLTQParserTest, DistributedExpandComponentTest, CloudExitableDirectoryReaderTest, TestPostingsSolrHighlighter, ResourceLoaderTest, TestBinaryResponseWriter, TestMergePolicyConfig, TestFiltering, LeaderInitiatedRecoveryOnCommitTest, TestCopyFieldCollectionResource, BadCopyFieldTest, TestStressReorder, TestLFUCache, TestCollapseQParserPlugin, TestCloudManagedSchemaConcurrent, DistributedQueryComponentOptimizationTest, LoggingHandlerTest, TestJsonFacets, TestBulkSchemaAPI, TestSchemaVersionResource, TestSchemaSimilarityResource, UUIDUpdateProcessorFallbackTest, TestObjectReleaseTracker, TestSolrIndexConfig, AnalyticsMergeStrategyTest, DeleteInactiveReplicaTest, SchemaVersionSpecificBehaviorTest, SpellCheckComponentTest, TestFileDictionaryLookup, TestConfigOverlay, TestExactSharedStatsCache, TestOrdValues, SpellingQueryConverterTest, TermVectorComponentDistributedTest, TestSolrConfigHandlerCloud, HdfsDirectoryFactoryTest, WordBreakSolrSpellCheckerTest, AssignTest, TestSearcherReuse, PrimUtilsTest, TestPartialUpdateDeduplication, UniqFieldsUpdateProcessorFactoryTest, TestCollectionAPI, TestDocBasedVersionConstraints, XmlUpdateRequestHandlerTest, BasicDistributedZk2Test, FieldAnalysisRequestHandlerTest, TestArbitraryIndexDir, ShowFileRequestHandlerTest, TestReplicationHandler, TestLRUStatsCache, CollectionsAPIDistributedZkTest, DistribDocExpirationUpdateProcessorTest, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J3 in 69.63s, 1 test, 1 error <<< FAILURES!

[...truncated 700 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 62 ignored (22 assumptions)

Total time: 53 minutes 55 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 205894437 bytes
Compression is 0.0%
Took 41 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2847 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2847/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:28199/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28199/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([B52BD4D3446D9A25:3D7FEB09EA91F7DD]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28767/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28767/collection1
	at __randomizedtesting.SeedInfo.seed([B52BD4D3446D9A25:3D7FEB09EA91F7DD]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28767/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:40397/repfacttest_c8n_1x3_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:40397/repfacttest_c8n_1x3_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([B52BD4D3446D9A25:3D7FEB09EA91F7DD]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:40397/repfacttest_c8n_1x3_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9781 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/init-core-data-001
   [junit4]   2> 757035 T967 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 757054 T967 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 757055 T968 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 757155 T967 oasc.ZkTestServer.run start zk server on port:28138
   [junit4]   2> 757156 T967 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 757158 T967 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 757166 T975 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@53ca89d name:ZooKeeperConnection Watcher:127.0.0.1:28138 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 757167 T967 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 757167 T967 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 757168 T967 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 757172 T967 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 757175 T967 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 757178 T978 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6450eadc name:ZooKeeperConnection Watcher:127.0.0.1:28138/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 757178 T967 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 757179 T967 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 757180 T967 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 757184 T967 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 757187 T967 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 757190 T967 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 757195 T967 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 757197 T967 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 757201 T967 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 757202 T967 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 757305 T967 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 757306 T967 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 757307 T967 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 757308 T967 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 757311 T967 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 757312 T967 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 757314 T967 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 757315 T967 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 757320 T967 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 757322 T967 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 757327 T967 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 757328 T967 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 757330 T967 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 757330 T967 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 757332 T967 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 757333 T967 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 757335 T967 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 757336 T967 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 762275 T967 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/control-001/cores/collection1
   [junit4]   2> 762280 T967 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 762286 T967 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:28166
   [junit4]   2> 762287 T967 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=28165, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/control-001/cores, hostContext=/}
   [junit4]   2> 762289 T967 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 762290 T967 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/control-001/'
   [junit4]   2> 762373 T967 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/control-001/solr.xml
   [junit4]   2> 762617 T967 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/control-001/cores
   [junit4]   2> 762617 T967 oasc.CoreContainer.<init> New CoreContainer 644736382
   [junit4]   2> 762618 T967 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/control-001/]
   [junit4]   2> 762618 T967 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/control-001/lib
   [junit4]   2> 762622 T967 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/control-001/lib).
   [junit4]   2> 762665 T967 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 762665 T967 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 762666 T967 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 762666 T967 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 762666 T967 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 762667 T967 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 762667 T967 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 762668 T967 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 762668 T967 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 762669 T967 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 762669 T967 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 762670 T967 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 762670 T967 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 762671 T967 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 762671 T967 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 762672 T967 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:28138/solr
   [junit4]   2> 762672 T967 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 762673 T967 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 762674 T967 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 762677 T990 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1a893a23 name:ZooKeeperConnection Watcher:127.0.0.1:28138 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 762678 T967 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 762679 T967 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 762687 T967 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 762690 T993 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7b9421fc name:ZooKeeperConnection Watcher:127.0.0.1:28138/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 762690 T967 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 762694 T967 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 762697 T967 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 762699 T967 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 762701 T967 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 762703 T967 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 762705 T967 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 762706 T967 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 762709 T967 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 762711 T967 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:28165_
   [junit4]   2> 762712 T967 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:28165_
   [junit4]   2> 762717 T967 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 762719 T967 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 762721 T967 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 762722 T967 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:28165_
   [junit4]   2> 762723 T967 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 762724 T967 oasc.Overseer.start Overseer (id=93553755579416579-127.0.0.1:28165_-n_0000000000) starting
   [junit4]   2> 762727 T967 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 762734 T967 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 762736 T995 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 762736 T967 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 762738 T994 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 762745 T967 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/control-001/cores
   [junit4]   2> 762755 T967 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {dataDir=data/, name=collection1, loadOnStartup=true, config=solrconfig.xml, schema=schema.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/control-001/cores/collection1/, coreNodeName=, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/control-001/cores/collection1, shard=, collection=control_collection}
   [junit4]   2> 762757 T967 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/control-001/cores/collection1/
   [junit4]   2> 762757 T967 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 762759 T997 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 762759 T997 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 762761 T997 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 762761 T993 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 762762 T994 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28165",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:28165_"} current state version: 0
   [junit4]   2> 762762 T994 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28165",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:28165_"}
   [junit4]   2> 762762 T994 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 762763 T994 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 762764 T993 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 763761 T997 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 763762 T997 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 763764 T997 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 763764 T997 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 763766 T997 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 763766 T997 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/control-001/cores/collection1/'
   [junit4]   2> 763808 T997 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 763818 T997 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 763883 T997 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 763959 T997 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 764099 T997 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 764201 T997 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 764233 T997 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 766154 T997 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 766226 T997 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 766236 T997 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 766260 T997 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 766273 T997 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 766284 T997 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 766286 T997 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 766287 T997 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 766287 T997 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 766290 T997 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 766290 T997 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 766291 T997 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 766291 T997 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 766292 T997 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/control-001/cores/collection1/data/
   [junit4]   2> 766292 T997 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@708f48a0
   [junit4]   2> 766294 T997 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/control-001/cores/collection1/data
   [junit4]   2> 766294 T997 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/control-001/cores/collection1/data/index/
   [junit4]   2> 766294 T997 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 766295 T997 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/control-001/cores/collection1/data/index
   [junit4]   2> 766296 T997 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=31, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1781859635902225]
   [junit4]   2> 766297 T997 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4b01fc27 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6f518ae9),segFN=segments_1,generation=1}
   [junit4]   2> 766297 T997 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 766305 T997 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 766305 T997 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 766306 T997 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 766306 T997 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 766307 T997 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 766307 T997 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 766308 T997 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 766309 T997 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 766309 T997 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 766437 T997 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 766466 T997 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 766474 T997 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 766479 T997 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 766489 T997 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update,/admin/properties,/admin/file,standard,/admin/mbeans,/admin/segments,/update/csv,/admin/logging,/admin/threads,/replication,/admin/ping,/update/json,/admin/system,/admin/luke,/config,/update/json/docs,/admin/plugins,/get,/schema
   [junit4]   2> 766520 T997 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 766525 T997 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 766526 T997 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 766530 T997 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 766531 T997 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 766532 T997 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=42, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3203903702708435]
   [junit4]   2> 766533 T997 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4b01fc27 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6f518ae9),segFN=segments_1,generation=1}
   [junit4]   2> 766534 T997 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 766534 T997 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@2d9bfd89[collection1] main
   [junit4]   2> 766535 T997 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 766536 T997 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 766536 T997 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 766537 T997 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 766538 T997 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 766538 T997 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 766539 T997 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 766539 T997 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 766540 T997 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 766540 T997 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 766541 T998 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2d9bfd89[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 766542 T997 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 766543 T1001 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:28165 collection:control_collection shard:shard1
   [junit4]   2> 766544 T967 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 766545 T967 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 766546 T1001 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 766551 T967 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 766552 T967 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 766554 T1005 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@26f5f093 name:ZooKeeperConnection Watcher:127.0.0.1:28138/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 766555 T967 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 766555 T967 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 766556 T967 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 766556 T1001 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 766558 T993 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 766558 T1001 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 766559 T1001 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C282 name=collection1 org.apache.solr.core.SolrCore@25b4127 url=http://127.0.0.1:28165/collection1 node=127.0.0.1:28165_ C282_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:28165, state=down, core=collection1, node_name=127.0.0.1:28165_}
   [junit4]   2> 766559 T1001 C:control_collection S:shard1 C282 P28165 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:28165/collection1/
   [junit4]   2> 766559 T994 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 766559 T1001 C:control_collection S:shard1 C282 P28165 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 766559 T967 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 766560 T1001 C:control_collection S:shard1 C282 P28165 oasc.SyncStrategy.syncToMe http://127.0.0.1:28165/collection1/ has no replicas
   [junit4]   2> 766560 T1001 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:28165/collection1/ shard1
   [junit4]   2> 766561 T1001 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 766562 T993 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 766562 T1005 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 766566 T993 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 766567 T994 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28165",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 766670 T1005 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 766670 T993 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 766718 T1001 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:28165/collection1/ and leader is http://127.0.0.1:28165/collection1/
   [junit4]   2> 766719 T1001 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:28165
   [junit4]   2> 766719 T1001 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 766719 T1001 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 766719 T1001 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 766721 T993 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 766722 T994 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28165",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:28165_"} current state version: 3
   [junit4]   2> 766722 T994 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28165",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:28165_"}
   [junit4]   2> 766826 T993 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 766826 T1005 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 771605 T967 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/shard-1-001/cores/collection1
   [junit4]   2> 771608 T967 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/shard-1-001
   [junit4]   2> 771609 T967 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 771612 T967 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:28200
   [junit4]   2> 771613 T967 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/shard-1-001/cores, hostPort=28199}
   [junit4]   2> 771613 T967 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 771614 T967 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/shard-1-001/'
   [junit4]   2> 771830 T967 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/shard-1-001/solr.xml
   [junit4]   2> 772151 T967 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/shard-1-001/cores
   [junit4]   2> 772151 T967 oasc.CoreContainer.<init> New CoreContainer 1518330454
   [junit4]   2> 772152 T967 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/shard-1-001/]
   [junit4]   2> 772152 T967 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/shard-1-001/lib
   [junit4]   2> 772154 T967 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/shard-1-001/lib).
   [junit4]   2> 772187 T967 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 772188 T967 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 772189 T967 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 772189 T967 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 772190 T967 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 772190 T967 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 772191 T967 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 772191 T967 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 772192 T967 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 772192 T967 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 772192 T967 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 772193 T967 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 772194 T967 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 772195 T967 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 772195 T967 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 772196 T967 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:28138/solr
   [junit4]   2> 772196 T967 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 772197 T967 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 772198 T967 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 772202 T1017 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@17ed2802 name:ZooKeeperConnection Watcher:127.0.0.1:28138 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 772204 T967 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 772205 T967 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 772208 T967 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 772210 T1020 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7697df13 name:ZooKeeperConnection Watcher:127.0.0.1:28138/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 772210 T967 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 772221 T967 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 773230 T967 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:28199_
   [junit4]   2> 773232 T967 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:28199_
   [junit4]   2> 773239 T967 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 773244 T967 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/shard-1-001/cores
   [junit4]   2> 773251 T967 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/shard-1-001/cores/collection1/, transient=false, schema=schema.xml, config=solrconfig.xml, name=collection1, dataDir=data/, coreNodeName=, shard=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/shard-1-001/cores/collection1, loadOnStartup=true, collection=collection1}
   [junit4]   2> 773252 T967 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/shard-1-001/cores/collection1/
   [junit4]   2> 773253 T967 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 773257 T1021 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 773257 T1021 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 773259 T993 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 773259 T1021 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 773260 T994 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28199",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:28199_"} current state version: 4
   [junit4]   2> 773261 T994 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28199",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:28199_"}
   [junit4]   2> 773261 T994 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 773262 T994 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 773263 T993 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 773264 T1005 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 773263 T1020 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 774260 T1021 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 774261 T1021 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 774261 T1021 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 774262 T1021 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 774262 T1021 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 774263 T1021 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/shard-1-001/cores/collection1/'
   [junit4]   2> 774348 T1021 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 774357 T1021 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 774414 T1021 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 774510 T1021 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 774703 T1021 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 774806 T1021 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 774823 T1021 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 776534 T1021 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 776597 T1021 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 776603 T1021 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 776630 T1021 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 776653 T1021 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 776660 T1021 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 776663 T1021 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 776663 T1021 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 776664 T1021 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 776665 T1021 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 776666 T1021 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 776666 T1021 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 776667 T1021 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 776667 T1021 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 776668 T1021 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@708f48a0
   [junit4]   2> 776671 T1021 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/shard-1-001/cores/collection1/data
   [junit4]   2> 776671 T1021 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 776671 T1021 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 776672 T1021 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B52BD4D3446D9A25-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 776673 T1021 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=31, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1781859635902225]
   [junit4]   2> 776675 T1021 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6908f764 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4ba9fc1b),segFN=segments_1,generation=1}
   [junit4]   2> 776675 T1021 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 776687 T1021 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 776688 T1021 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 776688 T1021 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 776688 T1021 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 776689 T1021 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 776689 T1021 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 776690 T1021 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 776690 T1021 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 776690 T1021 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 776779 T1021 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 776785 T1021 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 776790 T1021 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 776795 T1021 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 776807 T1021 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update,/admin/properties,/admin/file,standard,/admin/mbeans,/admin/segments,/update/csv,/admin/logging,/admin/threads,/replication,/admin/ping,/update/json,/admin/system,/admin/luke,/config,/update/json/docs,/admin/plugins,/get,/schema
   [junit4]   2> 776814 T1021 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 776819 T1021 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 776820 T1021 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 776827 T1021 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 776827 T1021 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 776830 T1021 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=42, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3203903702708435]
   [junit4]   2> 776831 T1021 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6908f764 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4ba9fc1b),segFN=segments_1,generation=1}
   [junit4]   2> 776831 T1021 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 776831 T1021 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@6d110db3[collection1] main
   [junit4]   2> 776832 T1021 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 776834 T1021 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 776834 T1021 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 776835 T1021 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 776836 T1021 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 776836 T1021 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 776838 T1021 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 776838 T1021 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 776838 T1021 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 776839 T1021 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 776840 T1022 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6d110db3[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 776841 T1021 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 776842 T1025 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:28199 collection:collection1 shard:shard1
   [junit4]   2> 776844 T967 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 776845 T967 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 776845 T1025 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 776857 T1025 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 776860 T993 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 776861 T1025 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 776861 T1025 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 776861 T994 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> ASYNC  NEW_CORE C283 name=collection1 org.apache.solr.core.SolrCore@6d527cfe url=http://127.0.0.1:28199/collection1 node=127.0.0.1:28199_ C283_STATE=coll:collection1 core:collection1 props:{base_url=http://127.0.0.1:28199, state=down, core=collection1, node_name=127.0.0.1:28199_}
   [junit4]   2> 776861 T1025 C:collection1 S:shard1 C283 P28199 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:28199/collection1/
   [junit4]   2> 776862 T1025 C:collection1 S:shard1 C283 P28199 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 776863 T1025 C:collection1 S:shard1 C283 P28199 oasc.SyncStrategy.syncToMe http://127.0.0.1:28199/collection1/ has no replicas
   [junit4]   2> 776863 T1025 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:28199/collection1/ shard1
   [junit4]   2> 776863 T1025 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 776864 T1020 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 776864 T993 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 776864 T1005 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 776868 T993 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 776869 T994 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28199",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 776973 T1005 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 776973 T993 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 776973 T1020 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 777021 T1025 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:28199/collection1/ and leader is http://127.0.0.1:28199/collection1/
   [junit4]   2> 777022 T1025 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:28199
   [junit4]   2> 777022 T1025 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 777022 T1025 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 777022 T1025 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 777024 T993 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 777025 T994 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28199",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:28199_"} current state version: 7
   [junit4]   2> 777026 T994 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28199",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:28199_"}
   [junit4]   2> 777131 T1020 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 777131 T993

[...truncated too long message...]


   [junit4]   2> 1953648 T4112 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:56060 56060
   [junit4]   2> 1953679 T4113 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:56060 56060
   [junit4]   2> 1953680 T4113 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1953681 T4112 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:61743/
   [junit4]   2> 1953681 T4112 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:51463/
   [junit4]   2> 1953681 T4112 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:28767/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=B52BD4D3446D9A25 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=es_AR -Dtests.timezone=America/Barbados -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   69.0s J2 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28767/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([B52BD4D3446D9A25:3D7FEB09EA91F7DD]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28767/collection1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 49 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 54 more
   [junit4]   2> 1953703 T4112 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest B52BD4D3446D9A25-001
   [junit4]   2> 69051 T4111 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarityProvider(queryNorm=true,coord=crazy): {}, locale=es_AR, timezone=America/Barbados
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=55801232,total=351797248
   [junit4]   2> NOTE: All tests run in this JVM: [TestNonDefinedSimilarityFactory, JsonLoaderTest, TestGroupingSearch, HdfsWriteToMultipleCollectionsTest, TestMiniSolrCloudCluster, DistributedQueryComponentCustomSortTest, AnalyticsQueryTest, SpellCheckComponentTest, TestStandardQParsers, QueryElevationComponentTest, TestSolrIndexConfig, DefaultValueUpdateProcessorTest, PluginInfoTest, CircularListTest, ResponseLogComponentTest, TestPerFieldSimilarity, TestJsonRequest, TestFieldTypeResource, TestCryptoKeys, TestOrdValues, TestSchemaManager, TestCloudSchemaless, CSVRequestHandlerTest, TestBlendedInfixSuggestions, MoreLikeThisHandlerTest, CopyFieldTest, SpellCheckCollatorTest, OutOfBoxZkACLAndCredentialsProvidersTest, TestCollationFieldDocValues, TestDocBasedVersionConstraints, TestReplicaProperties, TestManagedStopFilterFactory, SampleTest, CurrencyFieldOpenExchangeTest, TestCustomSort, TermsComponentTest, ScriptEngineTest, FullSolrCloudDistribCmdsTest, DocumentBuilderTest, TestRawResponseWriter, ShardRoutingCustomTest, TestJsonFacets, FieldAnalysisRequestHandlerTest, TestPseudoReturnFields, TestLRUCache, CloudMLTQParserTest, TestFieldResource, TestPhraseSuggestions, TestJmxMonitoredMap, TestPostingsSolrHighlighter, TestBadConfig, TestDistributedGrouping, StatsComponentTest, SortByFunctionTest, SuggestComponentTest, CollectionsAPIAsyncDistributedZkTest, ShowFileRequestHandlerTest, AddSchemaFieldsUpdateProcessorFactoryTest, UpdateRequestProcessorFactoryTest, LeaderFailoverAfterPartitionTest, TestReplicationHandler, MultiThreadedOCPTest, TestBlobHandler, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J2 in 69.33s, 1 test, 1 error <<< FAILURES!

[...truncated 702 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 62 ignored (22 assumptions)

Total time: 63 minutes 30 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 202146003 bytes
Compression is 0.0%
Took 41 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2846 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2846/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:40266/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:40266/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([8CECD24ACA3DAD82:4B8ED9064C1C07A]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([8CECD24ACA3DAD82:4B8ED9064C1C07A]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([8CECD24ACA3DAD82:4B8ED9064C1C07A]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9347 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/init-core-data-001
   [junit4]   2> 188719 T564 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 188724 T564 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 188724 T565 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 188824 T564 oasc.ZkTestServer.run start zk server on port:40800
   [junit4]   2> 188825 T564 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 188826 T564 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 188829 T572 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@55742246 name:ZooKeeperConnection Watcher:127.0.0.1:40800 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 188830 T564 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 188830 T564 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 188830 T564 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 188834 T564 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 188835 T564 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 188849 T575 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@25d50010 name:ZooKeeperConnection Watcher:127.0.0.1:40800/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 188850 T564 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 188850 T564 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 188850 T564 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 188853 T564 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 188856 T564 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 188858 T564 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 188861 T564 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 188861 T564 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 188865 T564 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 188866 T564 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 188869 T564 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 188870 T564 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 188872 T564 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 188873 T564 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 188876 T564 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 188877 T564 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 188879 T564 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 188880 T564 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 188883 T564 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 188883 T564 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 188886 T564 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 188887 T564 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 188890 T564 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 188890 T564 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 188893 T564 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 188894 T564 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 188896 T564 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 188897 T564 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 190253 T564 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/control-001/cores/collection1
   [junit4]   2> 190256 T564 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 190259 T564 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:48810
   [junit4]   2> 190260 T564 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/tempDir-001/control/data, hostContext=/, hostPort=10039}
   [junit4]   2> 190260 T564 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 190260 T564 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/control-001/'
   [junit4]   2> 190289 T564 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/control-001/solr.xml
   [junit4]   2> 190326 T564 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/control-001/cores
   [junit4]   2> 190326 T564 oasc.CoreContainer.<init> New CoreContainer 1940431588
   [junit4]   2> 190326 T564 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/control-001/]
   [junit4]   2> 190326 T564 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/control-001/lib
   [junit4]   2> 190327 T564 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/control-001/lib).
   [junit4]   2> 190336 T564 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 190336 T564 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 190336 T564 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 190336 T564 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 190337 T564 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 190337 T564 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 190337 T564 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 190337 T564 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 190337 T564 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 190338 T564 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 190338 T564 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 190338 T564 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 190339 T564 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 190339 T564 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 190339 T564 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 190339 T564 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:40800/solr
   [junit4]   2> 190339 T564 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 190340 T564 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 190340 T564 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 190342 T587 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@a661a67 name:ZooKeeperConnection Watcher:127.0.0.1:40800 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 190342 T564 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 190343 T564 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 190344 T564 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 190345 T590 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@525ff97c name:ZooKeeperConnection Watcher:127.0.0.1:40800/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 190345 T564 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 190347 T564 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 190349 T564 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 190351 T564 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 190352 T564 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 190354 T564 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 190356 T564 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 190358 T564 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 190359 T564 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 190360 T564 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:10039_
   [junit4]   2> 190361 T564 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:10039_
   [junit4]   2> 190362 T564 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 190364 T564 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 190365 T564 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 190366 T564 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:10039_
   [junit4]   2> 190366 T564 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 190368 T564 oasc.Overseer.start Overseer (id=93552765145317379-127.0.0.1:10039_-n_0000000000) starting
   [junit4]   2> 190370 T564 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 190375 T564 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 190376 T592 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 190376 T564 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 190376 T591 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 190380 T564 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/control-001/cores
   [junit4]   2> 190381 T564 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, dataDir=data/, shard=, collection=control_collection, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/control-001/cores/collection1, schema=schema.xml, coreNodeName=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/control-001/cores/collection1/, loadOnStartup=true, transient=false, name=collection1}
   [junit4]   2> 190382 T564 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/control-001/cores/collection1/
   [junit4]   2> 190382 T564 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 190383 T594 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 190383 T594 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 190384 T590 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 190384 T594 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 190384 T591 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:10039",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:10039_",
   [junit4]   2> 	  "operation":"state"} current state version: 0
   [junit4]   2> 190385 T591 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:10039",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:10039_",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 190385 T591 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 190385 T591 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 190386 T590 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 191384 T594 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 191385 T594 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 191385 T594 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 191386 T594 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 191387 T594 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 191387 T594 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/control-001/cores/collection1/'
   [junit4]   2> 191404 T594 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 191407 T594 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 191419 T594 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 191443 T594 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 191498 T594 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 191500 T594 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 191514 T594 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 191970 T594 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 191988 T594 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 191991 T594 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 192009 T594 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 192013 T594 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 192018 T594 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 192019 T594 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 192019 T594 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 192019 T594 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 192020 T594 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 192020 T594 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 192021 T594 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 192021 T594 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 192021 T594 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/control-001/cores/collection1/data/
   [junit4]   2> 192022 T594 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7cb9ded2
   [junit4]   2> 192023 T594 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/control-001/cores/collection1/data
   [junit4]   2> 192023 T594 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/control-001/cores/collection1/data/index/
   [junit4]   2> 192023 T594 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 192023 T594 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/control-001/cores/collection1/data/index
   [junit4]   2> 192024 T594 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=37, maxMergeAtOnceExplicit=33, maxMergedSegmentMB=59.1689453125, floorSegmentMB=0.814453125, forceMergeDeletesPctAllowed=27.33657082314279, segmentsPerTier=42.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 192025 T594 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3f6c0a70 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4e215f97),segFN=segments_1,generation=1}
   [junit4]   2> 192025 T594 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 192032 T594 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 192033 T594 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 192033 T594 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 192033 T594 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 192034 T594 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 192034 T594 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 192034 T594 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 192035 T594 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 192035 T594 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 192054 T594 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 192056 T594 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 192058 T594 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 192060 T594 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 192067 T594 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/segments,/get,/admin/ping,/replication,/update/json/docs,/schema,/update,/admin/properties,/update/csv,/admin/mbeans,/admin/luke,/admin/system,/admin/logging,/admin/plugins,/config,standard,/admin/threads,/update/json,/admin/file
   [junit4]   2> 192069 T594 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 192071 T594 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 192071 T594 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 192072 T594 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 192072 T594 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 192073 T594 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=21, maxMergeAtOnceExplicit=28, maxMergedSegmentMB=48.0322265625, floorSegmentMB=2.1044921875, forceMergeDeletesPctAllowed=28.176003449112116, segmentsPerTier=19.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.11702342794375618
   [junit4]   2> 192074 T594 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3f6c0a70 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4e215f97),segFN=segments_1,generation=1}
   [junit4]   2> 192074 T594 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 192075 T594 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@6cca9d97[collection1] main
   [junit4]   2> 192075 T594 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 192076 T594 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 192076 T594 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 192077 T594 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 192077 T594 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 192077 T594 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 192078 T594 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 192078 T594 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 192078 T594 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 192079 T594 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 192080 T595 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6cca9d97[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 192080 T594 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 192081 T598 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:10039 collection:control_collection shard:shard1
   [junit4]   2> 192081 T564 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 192082 T564 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 192082 T598 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 192085 T564 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 192085 T564 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 192087 T602 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@38b88117 name:ZooKeeperConnection Watcher:127.0.0.1:40800/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 192087 T564 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 192088 T564 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 192088 T564 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 192088 T598 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 192090 T590 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 192091 T598 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 192091 T598 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C204 name=collection1 org.apache.solr.core.SolrCore@332946f7 url=http://127.0.0.1:10039/collection1 node=127.0.0.1:10039_ C204_STATE=coll:control_collection core:collection1 props:{core=collection1, base_url=http://127.0.0.1:10039, state=down, node_name=127.0.0.1:10039_}
   [junit4]   2> 192091 T598 C:control_collection S:shard1 C204 P10039 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:10039/collection1/
   [junit4]   2> 192091 T598 C:control_collection S:shard1 C204 P10039 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 192091 T598 C:control_collection S:shard1 C204 P10039 oasc.SyncStrategy.syncToMe http://127.0.0.1:10039/collection1/ has no replicas
   [junit4]   2> 192092 T591 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 192092 T598 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:10039/collection1/ shard1
   [junit4]   2> 192092 T598 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 192092 T564 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 192093 T602 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 192093 T590 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 192099 T591 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:10039",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 192101 T590 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 192102 T602 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 192102 T590 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 192150 T598 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:10039/collection1/ and leader is http://127.0.0.1:10039/collection1/
   [junit4]   2> 192151 T598 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:10039
   [junit4]   2> 192151 T598 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 192151 T598 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 192151 T598 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 192153 T590 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 192154 T591 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:10039",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:10039_",
   [junit4]   2> 	  "operation":"state"} current state version: 3
   [junit4]   2> 192154 T591 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:10039",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:10039_",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 192258 T590 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 192259 T602 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 193190 T564 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/shard-1-001/cores/collection1
   [junit4]   2> 193192 T564 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/shard-1-001
   [junit4]   2> 193193 T564 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 193195 T564 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:15679
   [junit4]   2> 193195 T564 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=55705, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/shard-1-001/cores}
   [junit4]   2> 193196 T564 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 193196 T564 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/shard-1-001/'
   [junit4]   2> 193225 T564 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/shard-1-001/solr.xml
   [junit4]   2> 193261 T564 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/shard-1-001/cores
   [junit4]   2> 193261 T564 oasc.CoreContainer.<init> New CoreContainer 1541390205
   [junit4]   2> 193262 T564 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/shard-1-001/]
   [junit4]   2> 193262 T564 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/shard-1-001/lib
   [junit4]   2> 193262 T564 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/shard-1-001/lib).
   [junit4]   2> 193271 T564 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 193272 T564 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 193272 T564 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 193272 T564 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 193272 T564 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 193272 T564 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 193273 T564 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 193273 T564 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 193273 T564 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 193273 T564 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 193273 T564 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 193274 T564 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 193274 T564 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 193274 T564 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 193275 T564 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 193275 T564 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:40800/solr
   [junit4]   2> 193275 T564 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 193275 T564 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 193276 T564 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 193277 T614 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@23d07155 name:ZooKeeperConnection Watcher:127.0.0.1:40800 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 193278 T564 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 193278 T564 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 193280 T564 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 193281 T617 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@270780ac name:ZooKeeperConnection Watcher:127.0.0.1:40800/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 193281 T564 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 193286 T564 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 194291 T564 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55705_
   [junit4]   2> 194292 T564 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:55705_
   [junit4]   2> 194296 T564 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 194299 T564 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/shard-1-001/cores
   [junit4]   2> 194301 T564 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=collection1, config=solrconfig.xml, schema=schema.xml, coreNodeName=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/shard-1-001/cores/collection1, loadOnStartup=true, dataDir=data/, transient=false, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/shard-1-001/cores/collection1/, shard=, name=collection1}
   [junit4]   2> 194302 T564 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/shard-1-001/cores/collection1/
   [junit4]   2> 194302 T564 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 194303 T618 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 194303 T618 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 194304 T590 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 194304 T618 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 194305 T591 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:55705",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:55705_",
   [junit4]   2> 	  "operation":"state"} current state version: 4
   [junit4]   2> 194306 T591 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:55705",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:55705_",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 194306 T591 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 194306 T591 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 194307 T590 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 194307 T602 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 194307 T617 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 195304 T618 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 195305 T618 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 195305 T618 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 195306 T618 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 195307 T618 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 195307 T618 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/shard-1-001/cores/collection1/'
   [junit4]   2> 195324 T618 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 195327 T618 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 195339 T618 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 195363 T618 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 195433 T618 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 195434 T618 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 195448 T618 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 195967 T618 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 195979 T618 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 195981 T618 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 195995 T618 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 195998 T618 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 196001 T618 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 196002 T618 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 196002 T618 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 196002 T618 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 196003 T618 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 196003 T618 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 196004 T618 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 196004 T618 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 196004 T618 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 196004 T618 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7cb9ded2
   [junit4]   2> 196005 T618 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/shard-1-001/cores/collection1/data
   [junit4]   2> 196005 T618 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 196005 T618 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 196006 T618 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 196006 T618 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=37, maxMergeAtOnceExplicit=33, maxMergedSegmentMB=59.1689453125, floorSegmentMB=0.814453125, forceMergeDeletesPctAllowed=27.33657082314279, segmentsPerTier=42.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 196006 T618 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@c8f3299 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@d42e140),segFN=segments_1,generation=1}
   [junit4]   2> 196007 T618 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 196013 T618 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 196013 T618 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 196013 T618 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 196013 T618 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 196014 T618 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 196014 T618 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 196014 T618 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 196014 T618 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 196015 T618 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 196027 T618 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 196028 T618 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 196030 T618 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 196031 T618 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 196038 T618 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/segments,/get,/admin/ping,/replication,/update/json/docs,/schema,/update,/admin/properties,/update/csv,/admin/mbeans,/admin/luke,/admin/system,/admin/logging,/admin/plugins,/config,standard,/admin/threads,/update/json,/admin/file
   [junit4]   2> 196039 T618 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 196041 T618 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 196041 T618 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 196042 T618 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 196042 T618 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 196042 T618 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=21, maxMergeAtOnceExplicit=28, maxMergedSegmentMB=48.0322265625, floorSegmentMB=2.1044921875, forceMergeDeletesPctAllowed=28.176003449112116, segmentsPerTier=19.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.11702342794375618
   [junit4]   2> 196043 T618 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@c8f3299 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@d42e140),segFN=segments_1,generation=1}
   [junit4]   2> 196043 T618 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 196043 T618 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@8623808[collection1] main
   [junit4]   2> 196044 T618 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 196045 T618 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 196045 T618 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 196045 T618 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 196045 T618 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 196046 T618 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 196046 T618 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 196046 T618 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 196047 T618 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 196047 T618 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 196048 T619 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@8623808[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 196048 T618 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 196049 T622 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:55705 collection:collection1 shard:shard1
   [junit4]   2> 196050 T564 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 196050 T564 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 196050 T622 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 196056 T622 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 196058 T590 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 196058 T622 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 196059 T622 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C205 name=collection1 org.apache.solr.core.SolrCore@17ddc1c5 url=http://127.0.0.1:55705/collection1 node=127.0.0.1:55705_ C205_STATE=coll:collection1 core:collection1 props:{core=collection1, base_url=http://127.0.0.1:55705, state=down, node_name=127.0.0.1:55705_}
   [junit4]   2> 196059 T622 C:collection1 S:shard1 C205 P55705 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:55705/collection1/
   [junit4]   2> 196059 T622 C:collection1 S:shard1 C205 P55705 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 196059 T591 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 196059 T622 C:collection1 S:shard1 C205 P55705 oasc.SyncStrategy.syncToMe http://127.0.0.1:55705/collection1/ has no replicas
   [junit4]   2> 196060 T622 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:55705/collection1/ shard1
   [junit4]   2> 196060 T622 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 196061 T602 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 196061 T590 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 196061 T617 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 196066 T591 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:55705",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 196068 T590 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 196069 T590 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 196069 T602 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 196069 T617 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 196116 T622 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:55705/collection1/ and leader is http://127.0.0.1:55705/collection1/
   [junit4]   2> 196117 T622 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:55705
   [junit4]   2> 196117 T622 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 196117 T622 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 196117 T622 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 196118 T590 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 196119 T591 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:55705",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:55705_",
   [junit4]   2> 	  "operation":"state"} current state version: 7
   [junit4]   2> 196120 T591 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:55705",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:55705_",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 196224 T590 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 196224 T602 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 196224 T617 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 197214 T564 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/shard-2-001/cores/collection1
   [junit4]   2> 197216 T564 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 8CECD24ACA3DAD82-001/shard-2-001
   [junit4]   2> 197217 T564 oejs.Server.doSta

[...truncated too long message...]

	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/aliases.json
   [junit4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 		3	/solr/collections/c8n_1x3_commits/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 		5	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1544419 T4918 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 3 connections to: http://127.0.0.1:40281/
   [junit4]   2> 1544420 T4918 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:40266/
   [junit4]   2> 1544420 T4918 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:40293/
   [junit4]   2> 1544420 T4918 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:40253/
   [junit4]   2> 1544421 T4918 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 10 connections to: http://127.0.0.1:40262/
   [junit4]   2> 1544421 T4918 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:40281/
   [junit4]   2> 1544421 T4918 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:40266/
   [junit4]   2> 1544422 T4918 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:40293/
   [junit4]   2> 1544422 T4918 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:40253/
   [junit4]   2> 1544422 T4918 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:40262/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderInitiatedRecoveryOnCommitTest -Dtests.method=test -Dtests.seed=8CECD24ACA3DAD82 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=en_CA -Dtests.timezone=Europe/Kaliningrad -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   79.4s J3 | LeaderInitiatedRecoveryOnCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:40266/c8n_1x3_commits_shard1_replica3
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([8CECD24ACA3DAD82:4B8ED9064C1C07A]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 48 more
   [junit4]   2> 1544440 T4918 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8CECD24ACA3DAD82-001
   [junit4]   2> 79440 T4917 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=HighCompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=HIGH_COMPRESSION, chunkSize=23668, maxDocsPerChunk=5, blockSize=578), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=HIGH_COMPRESSION, chunkSize=23668, blockSize=578)), sim=RandomSimilarityProvider(queryNorm=false,coord=no): {}, locale=en_CA, timezone=Europe/Kaliningrad
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=111212152,total=328204288
   [junit4]   2> NOTE: All tests run in this JVM: [IndexSchemaTest, ClusterStateTest, ChaosMonkeySafeLeaderTest, ActionThrottleTest, TestReplicationHandlerBackup, SoftAutoCommitTest, TestInitParams, UUIDFieldTest, SimplePostToolTest, TestTrie, CoreMergeIndexesAdminHandlerTest, BJQParserTest, TestTolerantSearch, TestExactSharedStatsCache, TestSolrDeletionPolicy2, SliceStateTest, TestManagedSchema, FieldMutatingUpdateProcessorTest, SuggesterTest, TestLRUCache, CustomCollectionTest, CoreAdminCreateDiscoverTest, OverseerCollectionProcessorTest, TestCollationField, StandardRequestHandlerTest, TestQueryUtils, SpatialHeatmapFacetsTest, TestMiniSolrCloudCluster, SolrXmlInZkTest, RecoveryZkTest, RankQueryTest, DistributedTermsComponentTest, BasicDistributedZk2Test, IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestManagedResourceStorage, ExitableDirectoryReaderTest, TestCloudPivotFacet, TestUtils, DistanceFunctionTest, UUIDUpdateProcessorFallbackTest, TestLMDirichletSimilarityFactory, DistributedFacetPivotWhiteBoxTest, SimpleFacetsTest, RegexBoostProcessorTest, TestFieldSortValues, SpellPossibilityIteratorTest, TestJoin, TestZkChroot, ClusterStateUpdateTest, ConvertedLegacyTest, CursorPagingTest, RequestHandlersTest, TestRTGBase, ShardRoutingTest, SchemaVersionSpecificBehaviorTest, TestReplicationHandler, BlockCacheTest, RAMDirectoryFactoryTest, TestSolrQueryParserDefaultOperatorResource, ShardSplitTest, LeaderInitiatedRecoveryOnCommitTest]
   [junit4] Completed on J3 in 79.99s, 1 test, 1 error <<< FAILURES!

[...truncated 717 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 62 ignored (22 assumptions)

Total time: 53 minutes 35 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 214650566 bytes
Compression is 0.0%
Took 39 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2845 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2845/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:37664/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37664/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([3C00720EB53D0BA0:B4544DD41BC16658]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:32922/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:32922/collection1
	at __randomizedtesting.SeedInfo.seed([3C00720EB53D0BA0:B4544DD41BC16658]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:32922/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:59104/pv_hy/repfacttest_c8n_1x3_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:59104/pv_hy/repfacttest_c8n_1x3_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([3C00720EB53D0BA0:B4544DD41BC16658]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:59104/pv_hy/repfacttest_c8n_1x3_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9555 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/init-core-data-001
   [junit4]   2> 557305 T1542 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 557311 T1542 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 557312 T1543 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 557412 T1542 oasc.ZkTestServer.run start zk server on port:32910
   [junit4]   2> 557413 T1542 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 557414 T1542 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 557417 T1550 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7343568a name:ZooKeeperConnection Watcher:127.0.0.1:32910 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 557418 T1542 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 557418 T1542 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 557418 T1542 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 557421 T1542 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 557422 T1542 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 557423 T1553 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7e7c52f6 name:ZooKeeperConnection Watcher:127.0.0.1:32910/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 557423 T1542 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 557424 T1542 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 557424 T1542 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 557425 T1542 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 557427 T1542 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 557428 T1542 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 557430 T1542 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 557430 T1542 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 557432 T1542 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 557433 T1542 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 557535 T1542 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 557536 T1542 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 557537 T1542 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 557538 T1542 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 557539 T1542 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 557540 T1542 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 557542 T1542 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 557542 T1542 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 557544 T1542 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 557544 T1542 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 557546 T1542 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 557546 T1542 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 557548 T1542 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 557549 T1542 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 557550 T1542 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 557551 T1542 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 557552 T1542 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 557553 T1542 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 558694 T1542 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/control-001/cores/collection1
   [junit4]   2> 558697 T1542 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 558700 T1542 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:32914
   [junit4]   2> 558700 T1542 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=32913, hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/tempDir-001/control/data}
   [junit4]   2> 558701 T1542 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@5fb271f1
   [junit4]   2> 558701 T1542 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/control-001/'
   [junit4]   2> 558730 T1542 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/control-001/solr.xml
   [junit4]   2> 558765 T1542 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/control-001/cores
   [junit4]   2> 558765 T1542 oasc.CoreContainer.<init> New CoreContainer 1639142761
   [junit4]   2> 558765 T1542 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/control-001/]
   [junit4]   2> 558765 T1542 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/control-001/lib
   [junit4]   2> 558766 T1542 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/control-001/lib).
   [junit4]   2> 558775 T1542 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 558775 T1542 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 558775 T1542 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 558775 T1542 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 558775 T1542 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 558776 T1542 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 558776 T1542 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 558776 T1542 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 558776 T1542 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 558776 T1542 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 558777 T1542 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 558777 T1542 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 558777 T1542 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 558777 T1542 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 558778 T1542 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 558778 T1542 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:32910/solr
   [junit4]   2> 558778 T1542 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 558778 T1542 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 558779 T1542 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 558780 T1565 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6e5c2253 name:ZooKeeperConnection Watcher:127.0.0.1:32910 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 558781 T1542 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 558781 T1542 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 558783 T1542 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 558784 T1568 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4e52e9bc name:ZooKeeperConnection Watcher:127.0.0.1:32910/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 558784 T1542 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 558785 T1542 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 558787 T1542 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 558788 T1542 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 558789 T1542 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 558790 T1542 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 558792 T1542 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 558793 T1542 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 558794 T1542 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 558795 T1542 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:32913_
   [junit4]   2> 558795 T1542 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:32913_
   [junit4]   2> 558796 T1542 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 558797 T1542 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 558798 T1542 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 558799 T1542 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:32913_
   [junit4]   2> 558799 T1542 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 558800 T1542 oasc.Overseer.start Overseer (id=93551844656218115-127.0.0.1:32913_-n_0000000000) starting
   [junit4]   2> 558802 T1542 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 558806 T1542 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 558806 T1570 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 558806 T1569 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 558806 T1542 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 558809 T1542 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/control-001/cores
   [junit4]   2> 558811 T1542 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/control-001/cores/collection1/, dataDir=data/, schema=schema.xml, config=solrconfig.xml, name=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/control-001/cores/collection1, transient=false, shard=, loadOnStartup=true, collection=control_collection, coreNodeName=}
   [junit4]   2> 558811 T1542 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/control-001/cores/collection1/
   [junit4]   2> 558811 T1542 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 558812 T1572 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 558812 T1572 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 558813 T1568 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 558813 T1572 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 558813 T1569 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:32913_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32913",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state"} current state version: 0
   [junit4]   2> 558814 T1569 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:32913_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32913",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 558814 T1569 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 558814 T1569 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 558815 T1568 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 559813 T1572 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 559814 T1572 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 559814 T1572 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 559815 T1572 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 559815 T1572 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 559815 T1572 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/control-001/cores/collection1/'
   [junit4]   2> 559832 T1572 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 559835 T1572 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 559845 T1572 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 559867 T1572 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 559915 T1572 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 560017 T1572 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 560028 T1572 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 560424 T1572 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 560439 T1572 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 560442 T1572 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 560451 T1572 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 560455 T1572 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 560459 T1572 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 560460 T1572 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 560460 T1572 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 560460 T1572 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 560461 T1572 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 560461 T1572 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 560461 T1572 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 560461 T1572 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 560462 T1572 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/control-001/cores/collection1/data/
   [junit4]   2> 560462 T1572 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1a2784fd
   [junit4]   2> 560471 T1572 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/control-001/cores/collection1/data
   [junit4]   2> 560472 T1572 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/control-001/cores/collection1/data/index/
   [junit4]   2> 560472 T1572 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 560472 T1572 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/control-001/cores/collection1/data/index
   [junit4]   2> 560476 T1572 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@7737490d
   [junit4]   2> 560477 T1572 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4e9d31a0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@322527b2),segFN=segments_1,generation=1}
   [junit4]   2> 560477 T1572 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 560483 T1572 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 560483 T1572 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 560484 T1572 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 560484 T1572 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 560484 T1572 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 560484 T1572 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 560485 T1572 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 560485 T1572 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 560485 T1572 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 560504 T1572 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 560506 T1572 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 560508 T1572 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 560511 T1572 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 560516 T1572 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update/json/docs,/admin/mbeans,/admin/system,standard,/admin/plugins,/admin/properties,/admin/threads,/admin/ping,/schema,/admin/luke,/admin/segments,/update/csv,/admin/file,/config,/get,/replication,/update/json,/admin/logging,/update
   [junit4]   2> 560518 T1572 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 560519 T1572 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 560520 T1572 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 560521 T1572 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 560521 T1572 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 560522 T1572 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=43, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=23.7978515625, floorSegmentMB=0.6474609375, forceMergeDeletesPctAllowed=16.151172565014342, segmentsPerTier=39.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.288499120953874
   [junit4]   2> 560523 T1572 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4e9d31a0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@322527b2),segFN=segments_1,generation=1}
   [junit4]   2> 560523 T1572 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 560523 T1572 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@2abc1f06[collection1] main
   [junit4]   2> 560524 T1572 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 560524 T1572 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 560525 T1572 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 560525 T1572 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 560525 T1572 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 560525 T1572 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 560526 T1572 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 560526 T1572 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 560526 T1572 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 560527 T1572 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 560527 T1573 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2abc1f06[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 560528 T1572 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 560529 T1576 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:32913 collection:control_collection shard:shard1
   [junit4]   2> 560529 T1542 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 560529 T1542 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 560529 T1576 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 560532 T1542 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 560533 T1576 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 560533 T1542 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 560534 T1568 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 560534 T1576 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 560535 T1576 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 560535 T1580 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2a2a358 name:ZooKeeperConnection Watcher:127.0.0.1:32910/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 560535 T1569 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C261 name=collection1 org.apache.solr.core.SolrCore@31131b0e url=http://127.0.0.1:32913/collection1 node=127.0.0.1:32913_ C261_STATE=coll:control_collection core:collection1 props:{state=down, node_name=127.0.0.1:32913_, base_url=http://127.0.0.1:32913, core=collection1}
   [junit4]   2> 560535 T1576 C:control_collection S:shard1 C261 P32913 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:32913/collection1/
   [junit4]   2> 560535 T1542 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 560535 T1576 C:control_collection S:shard1 C261 P32913 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 560536 T1576 C:control_collection S:shard1 C261 P32913 oasc.SyncStrategy.syncToMe http://127.0.0.1:32913/collection1/ has no replicas
   [junit4]   2> 560536 T1576 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:32913/collection1/ shard1
   [junit4]   2> 560536 T1568 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 560536 T1542 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 560536 T1576 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 560537 T1542 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 560540 T1542 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 560540 T1568 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 560541 T1569 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32913",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 560643 T1568 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 560643 T1580 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 560692 T1576 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:32913/collection1/ and leader is http://127.0.0.1:32913/collection1/
   [junit4]   2> 560693 T1576 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:32913
   [junit4]   2> 560693 T1576 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 560693 T1576 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 560693 T1576 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 560694 T1568 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 560695 T1569 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:32913_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32913",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state"} current state version: 3
   [junit4]   2> 560695 T1569 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:32913_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32913",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 560798 T1568 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 560798 T1580 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 561669 T1542 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/shard-1-001/cores/collection1
   [junit4]   2> 561671 T1542 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/shard-1-001
   [junit4]   2> 561672 T1542 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 561674 T1542 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:32923
   [junit4]   2> 561674 T1542 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solrconfig=solrconfig.xml, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/shard-1-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/tempDir-001/jetty1, hostContext=/, hostPort=32922}
   [junit4]   2> 561675 T1542 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@5fb271f1
   [junit4]   2> 561675 T1542 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/shard-1-001/'
   [junit4]   2> 561703 T1542 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/shard-1-001/solr.xml
   [junit4]   2> 561738 T1542 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/shard-1-001/cores
   [junit4]   2> 561738 T1542 oasc.CoreContainer.<init> New CoreContainer 1071039309
   [junit4]   2> 561738 T1542 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/shard-1-001/]
   [junit4]   2> 561738 T1542 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/shard-1-001/lib
   [junit4]   2> 561739 T1542 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/shard-1-001/lib).
   [junit4]   2> 561748 T1542 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 561748 T1542 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 561748 T1542 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 561748 T1542 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 561749 T1542 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 561749 T1542 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 561749 T1542 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 561749 T1542 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 561749 T1542 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 561750 T1542 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 561750 T1542 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 561750 T1542 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 561751 T1542 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 561751 T1542 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 561751 T1542 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 561751 T1542 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:32910/solr
   [junit4]   2> 561751 T1542 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 561752 T1542 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 561752 T1542 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 561754 T1592 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6c77248a name:ZooKeeperConnection Watcher:127.0.0.1:32910 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 561754 T1542 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 561755 T1542 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 561756 T1542 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 561757 T1595 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@e18e5da name:ZooKeeperConnection Watcher:127.0.0.1:32910/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 561758 T1542 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 561761 T1542 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 562763 T1542 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:32922_
   [junit4]   2> 562764 T1542 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:32922_
   [junit4]   2> 562767 T1542 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 562769 T1542 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/shard-1-001/cores
   [junit4]   2> 562771 T1542 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {coreNodeName=, shard=, collection=collection1, config=solrconfig.xml, name=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/shard-1-001/cores/collection1/, dataDir=data/, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/shard-1-001/cores/collection1, schema=schema.xml, loadOnStartup=true}
   [junit4]   2> 562771 T1542 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/shard-1-001/cores/collection1/
   [junit4]   2> 562771 T1542 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 562772 T1596 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 562772 T1596 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 562773 T1568 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 562773 T1596 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 562774 T1569 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:32922_",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32922",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state"} current state version: 4
   [junit4]   2> 562774 T1569 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:32922_",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32922",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 562774 T1569 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 562775 T1569 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 562776 T1595 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 562776 T1568 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 562776 T1580 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 563773 T1596 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 563774 T1596 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 563774 T1596 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 563774 T1596 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 563775 T1596 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 563775 T1596 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/shard-1-001/cores/collection1/'
   [junit4]   2> 563787 T1596 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 563789 T1596 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 563796 T1596 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 564076 T1596 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 564109 T1596 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 564210 T1596 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 564217 T1596 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 564515 T1596 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 564530 T1596 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 564533 T1596 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 564543 T1596 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 564548 T1596 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 564551 T1596 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 564552 T1596 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 564553 T1596 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 564553 T1596 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 564554 T1596 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 564554 T1596 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 564554 T1596 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 564554 T1596 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 564555 T1596 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 564555 T1596 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1a2784fd
   [junit4]   2> 564558 T1596 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/shard-1-001/cores/collection1/data
   [junit4]   2> 564558 T1596 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 564559 T1596 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 564559 T1596 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 564559 T1596 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@7381ab33
   [junit4]   2> 564560 T1596 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@460edc83 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@79af4063),segFN=segments_1,generation=1}
   [junit4]   2> 564561 T1596 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 564568 T1596 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 564568 T1596 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 564568 T1596 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 564569 T1596 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 564569 T1596 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 564569 T1596 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 564570 T1596 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 564570 T1596 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 564570 T1596 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 564589 T1596 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 564591 T1596 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 564593 T1596 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 564595 T1596 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 564603 T1596 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update/json/docs,/admin/mbeans,/admin/system,standard,/admin/plugins,/admin/properties,/admin/threads,/admin/ping,/schema,/admin/luke,/admin/segments,/update/csv,/admin/file,/config,/get,/replication,/update/json,/admin/logging,/update
   [junit4]   2> 564606 T1596 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 564607 T1596 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 564607 T1596 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 564609 T1596 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 564609 T1596 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 564610 T1596 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=43, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=23.7978515625, floorSegmentMB=0.6474609375, forceMergeDeletesPctAllowed=16.151172565014342, segmentsPerTier=39.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.288499120953874
   [junit4]   2> 564611 T1596 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@460edc83 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@79af4063),segFN=segments_1,generation=1}
   [junit4]   2> 564611 T1596 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 564612 T1596 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@56bdbbec[collection1] main
   [junit4]   2> 564612 T1596 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 564613 T1596 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 564613 T1596 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 564613 T1596 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 564614 T1596 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 564614 T1596 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 564614 T1596 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 564615 T1596 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 564615 T1596 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 564615 T1596 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 564616 T1597 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@56bdbbec[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 564617 T1596 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 564617 T1600 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:32922 collection:collection1 shard:shard1
   [junit4]   2> 564617 T1542 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 564618 T1542 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 564618 T1600 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 564622 T1600 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 564623 T1568 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 564623 T1600 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 564623 T1600 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C262 name=collection1 org.apache.solr.core.SolrCore@9d0d49e url=http://127.0.0.1:32922/collection1 node=127.0.0.1:32922_ C262_STATE=coll:collection1 core:collection1 props:{state=down, node_name=127.0.0.1:32922_, base_url=http://127.0.0.1:32922, core=collection1}
   [junit4]   2> 564623 T1600 C:collection1 S:shard1 C262 P32922 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:32922/collection1/
   [junit4]   2> 564624 T1569 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 564624 T1600 C:collection1 S:shard1 C262 P32922 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 564624 T1600 C:collection1 S:shard1 C262 P32922 oasc.SyncStrategy.syncToMe http://127.0.0.1:32922/collection1/ has no replicas
   [junit4]   2> 564624 T1600 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:32922/collection1/ shard1
   [junit4]   2> 564625 T1600 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 564625 T1568 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 564625 T1580 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 564625 T1595 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 564628 T1568 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 564629 T1569 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32922",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 564732 T1580 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 564732 T1595 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 564732 T1568 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 564781 T1600 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:32922/collection1/ and leader is http://127.0.0.1:32922/collection1/
   [junit4]   2> 564782 T1600 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:32922
   [junit4]   2> 564782 T1600 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 564782 T1600 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 564782 T1600 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 564783 T1568 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 564784 T1569 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:32922_",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32922",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state"} current state version: 7
   [junit4]   2> 564785 T1569 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:32922_",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32922",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 564888 T1595 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 564888 T1568 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 564888 T1580 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 565530 T1542 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 3C00720EB53D0BA0-001/shard-2-001/cores

[...truncated too long message...]

erWord connecting to 127.0.0.1:16438 16438
   [junit4]   2> 1560668 T3881 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/aliases.json
   [junit4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 		3	/solr/collections/c8n_1x3_commits/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent data watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		2	/solr/collections/c8n_1x3_commits/leader_elect/shard1/election/93551905304608780-core_node3-n_0000000000
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 		4	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1560669 T3880 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 3 connections to: http://127.0.0.1:50890/
   [junit4]   2> 1560669 T3880 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:37664/
   [junit4]   2> 1560670 T3880 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:37697/
   [junit4]   2> 1560670 T3880 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:37683/
   [junit4]   2> 1560670 T3880 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 9 connections to: http://127.0.0.1:37644/
   [junit4]   2> 1560671 T3880 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:50890/
   [junit4]   2> 1560671 T3880 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:37664/
   [junit4]   2> 1560671 T3880 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:37697/
   [junit4]   2> 1560672 T3880 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:37683/
   [junit4]   2> 1560672 T3880 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:37644/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderInitiatedRecoveryOnCommitTest -Dtests.method=test -Dtests.seed=3C00720EB53D0BA0 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=pl_PL -Dtests.timezone=CST6CDT -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   78.7s J2 | LeaderInitiatedRecoveryOnCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37664/c8n_1x3_commits_shard1_replica2
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([3C00720EB53D0BA0:B4544DD41BC16658]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 48 more
   [junit4]   2> 1560688 T3880 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 3C00720EB53D0BA0-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=DefaultSimilarity, locale=pl_PL, timezone=CST6CDT
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=80517808,total=352321536
   [junit4]   2> NOTE: All tests run in this JVM: [TestReplicaProperties, HdfsRecoveryZkTest, SuggesterTSTTest, PreAnalyzedUpdateProcessorTest, TestReRankQParserPlugin, ScriptEngineTest, TestArbitraryIndexDir, TestPartialUpdateDeduplication, DocValuesMissingTest, TestCodecSupport, TestNRTOpen, DirectUpdateHandlerTest, TestConfigOverlay, AddSchemaFieldsUpdateProcessorFactoryTest, SolrIndexSplitterTest, TestInitParams, TestStressReorder, LeaderElectionIntegrationTest, SolrIndexConfigTest, TestJettySolrRunner, TestStressRecovery, ExitableDirectoryReaderTest, TestUniqueKeyFieldResource, AsyncMigrateRouteKeyTest, SyncSliceTest, DocValuesTest, PrimitiveFieldTypeTest, TestSolrConfigHandler, SSLMigrationTest, OpenExchangeRatesOrgProviderTest, TestFastWriter, EchoParamsTest, TestRandomMergePolicy, TestManagedSchema, FullSolrCloudDistribCmdsTest, IndexSchemaTest, DistribDocExpirationUpdateProcessorTest, TestOverriddenPrefixQueryForCustomFieldType, PrimUtilsTest, TestIndexSearcher, TestStressVersions, TestCloudSchemaless, TestOmitPositions, TestDefaultSearchFieldResource, TestDFRSimilarityFactory, ResponseLogComponentTest, HdfsSyncSliceTest, DeleteLastCustomShardedReplicaTest, TestOrdValues, TestNamedUpdateProcessors, TestDefaultStatsCache, ResourceLoaderTest, TestManagedSynonymFilterFactory, TestReplicationHandlerBackup, TestRealTimeGet, PreAnalyzedFieldTest, TestDynamicFieldResource, StandardRequestHandlerTest, ShardSplitTest, CollectionsAPIDistributedZkTest, LeaderInitiatedRecoveryOnCommitTest]
   [junit4] Completed on J2 in 78.72s, 1 test, 1 error <<< FAILURES!

[...truncated 716 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 62 ignored (22 assumptions)

Total time: 52 minutes 54 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 214878998 bytes
Compression is 0.0%
Took 41 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2844 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2844/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:56434/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:56434/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([888997ABE0D7069C:DDA8714E2B6B64]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:25079/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:25079/collection1
	at __randomizedtesting.SeedInfo.seed([888997ABE0D7069C:DDA8714E2B6B64]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:25079/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:26423/nhff/w/repfacttest_c8n_1x3_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:26423/nhff/w/repfacttest_c8n_1x3_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([888997ABE0D7069C:DDA8714E2B6B64]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:26423/nhff/w/repfacttest_c8n_1x3_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9269 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/init-core-data-001
   [junit4]   2> 57859 T100 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 57895 T100 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 57903 T101 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 58204 T100 oasc.ZkTestServer.run start zk server on port:56393
   [junit4]   2> 58379 T100 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 58623 T100 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 58848 T108 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@24bcd560 name:ZooKeeperConnection Watcher:127.0.0.1:56393 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 58849 T100 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 58850 T100 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 58870 T100 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 59070 T100 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 59071 T100 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 59073 T111 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2677d1e3 name:ZooKeeperConnection Watcher:127.0.0.1:56393/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 59074 T100 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 59075 T100 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 59106 T100 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 59112 T100 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 59116 T100 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 59119 T100 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 59124 T100 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 59125 T100 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 59132 T100 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 59133 T100 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 59238 T100 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 59239 T100 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 59244 T100 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 59245 T100 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 59249 T100 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 59250 T100 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 59255 T100 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 59256 T100 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 59260 T100 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 59261 T100 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 59265 T100 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 59266 T100 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 59271 T100 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 59272 T100 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 59276 T100 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 59277 T100 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 59281 T100 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 59282 T100 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 61207 T100 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/control-001/cores/collection1
   [junit4]   2> 61212 T100 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 61215 T100 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:56406
   [junit4]   2> 61215 T100 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/tempDir-001/control/data, hostContext=/, hostPort=56405}
   [junit4]   2> 61216 T100 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 61217 T100 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/control-001/'
   [junit4]   2> 61271 T100 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/control-001/solr.xml
   [junit4]   2> 61341 T100 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/control-001/cores
   [junit4]   2> 61342 T100 oasc.CoreContainer.<init> New CoreContainer 2095349609
   [junit4]   2> 61343 T100 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/control-001/]
   [junit4]   2> 61343 T100 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/control-001/lib
   [junit4]   2> 61344 T100 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/control-001/lib).
   [junit4]   2> 61355 T100 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 61355 T100 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 61356 T100 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 61356 T100 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 61356 T100 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 61357 T100 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 61357 T100 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 61357 T100 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 61358 T100 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 61358 T100 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 61358 T100 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 61359 T100 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 61359 T100 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 61360 T100 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 61360 T100 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 61361 T100 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:56393/solr
   [junit4]   2> 61432 T100 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 61432 T100 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 61433 T100 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 61436 T123 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@67f4e328 name:ZooKeeperConnection Watcher:127.0.0.1:56393 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 61437 T100 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 61437 T100 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 61461 T100 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 61464 T126 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@69298d3e name:ZooKeeperConnection Watcher:127.0.0.1:56393/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 61464 T100 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 61488 T100 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 61507 T100 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 61519 T100 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 61523 T100 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 61527 T100 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 61563 T100 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 61566 T100 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 61569 T100 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 61571 T100 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56405_
   [junit4]   2> 61581 T100 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:56405_
   [junit4]   2> 61585 T100 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 61588 T100 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 61591 T100 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 61631 T100 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:56405_
   [junit4]   2> 61632 T100 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 61636 T100 oasc.Overseer.start Overseer (id=93551095708778499-127.0.0.1:56405_-n_0000000000) starting
   [junit4]   2> 61662 T100 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 61697 T100 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 61725 T128 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 61725 T100 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 61758 T127 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 61765 T100 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/control-001/cores
   [junit4]   2> 61787 T100 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, dataDir=data/, shard=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/control-001/cores/collection1/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/control-001/cores/collection1, name=collection1, coreNodeName=, transient=false, schema=schema.xml, loadOnStartup=true, collection=control_collection}
   [junit4]   2> 61788 T100 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/control-001/cores/collection1/
   [junit4]   2> 61788 T100 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 61789 T130 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 61790 T130 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 61810 T126 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 61810 T130 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 61819 T127 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:56405",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:56405_",
   [junit4]   2> 	  "state":"down"} current state version: 0
   [junit4]   2> 61885 T127 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:56405",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:56405_",
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 61893 T127 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 61949 T127 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 61970 T126 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 62812 T130 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 62812 T130 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 62814 T130 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 62814 T130 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 62816 T130 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 62817 T130 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/control-001/cores/collection1/'
   [junit4]   2> 62873 T130 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 62900 T130 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 62943 T130 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 62978 T130 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 63044 T130 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 63147 T130 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 63203 T130 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 63742 T130 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 63769 T130 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 63773 T130 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 63810 T130 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 63817 T130 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 63824 T130 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 63826 T130 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 63827 T130 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 63827 T130 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 63829 T130 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 63829 T130 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 63830 T130 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 63830 T130 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 63831 T130 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/control-001/cores/collection1/data/
   [junit4]   2> 63831 T130 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@73f2f11
   [junit4]   2> 63852 T130 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/control-001/cores/collection1/data
   [junit4]   2> 63852 T130 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/control-001/cores/collection1/data/index/
   [junit4]   2> 63853 T130 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 63853 T130 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/control-001/cores/collection1/data/index
   [junit4]   2> 63854 T130 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=13, maxMergeAtOnceExplicit=28, maxMergedSegmentMB=78.78125, floorSegmentMB=0.6962890625, forceMergeDeletesPctAllowed=6.197528975712406, segmentsPerTier=39.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 63855 T130 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@17cf1302 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1be8f484),segFN=segments_1,generation=1}
   [junit4]   2> 63856 T130 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 63869 T130 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 63971 T130 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 63972 T130 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 63972 T130 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 63973 T130 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 63973 T130 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 63974 T130 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 63975 T130 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 63975 T130 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 64003 T130 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 64017 T130 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 64020 T130 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 64024 T130 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 64041 T130 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/logging,/get,/update/csv,/admin/plugins,/admin/luke,standard,/admin/file,/update,/admin/threads,/admin/system,/admin/ping,/admin/segments,/admin/properties,/replication,/admin/mbeans,/update/json,/config,/schema,/update/json/docs
   [junit4]   2> 64044 T130 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 64047 T130 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 64047 T130 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 64049 T130 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 64049 T130 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 64052 T130 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=47, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6096064083982916]
   [junit4]   2> 64053 T130 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@17cf1302 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1be8f484),segFN=segments_1,generation=1}
   [junit4]   2> 64053 T130 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 64054 T130 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@5df7a803[collection1] main
   [junit4]   2> 64054 T130 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 64056 T130 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 64057 T130 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 64074 T130 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 64074 T130 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 64075 T130 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 64076 T130 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 64076 T130 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 64077 T130 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 64077 T130 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 64081 T131 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5df7a803[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 64090 T130 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 64091 T134 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:56405 collection:control_collection shard:shard1
   [junit4]   2> 64092 T100 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 64092 T100 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 64136 T134 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 64146 T134 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 64147 T100 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 64148 T100 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 64149 T126 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 64150 T138 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@29d5f860 name:ZooKeeperConnection Watcher:127.0.0.1:56393/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 64150 T134 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 64150 T100 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 64151 T134 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 64152 T100 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> ASYNC  NEW_CORE C9 name=collection1 org.apache.solr.core.SolrCore@801d06 url=http://127.0.0.1:56405/collection1 node=127.0.0.1:56405_ C9_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:56405, core=collection1, node_name=127.0.0.1:56405_, state=down}
   [junit4]   2> 64152 T134 C:control_collection S:shard1 C9 P56405 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:56405/collection1/
   [junit4]   2> 64152 T127 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 64153 T134 C:control_collection S:shard1 C9 P56405 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 64152 T100 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 64154 T134 C:control_collection S:shard1 C9 P56405 oasc.SyncStrategy.syncToMe http://127.0.0.1:56405/collection1/ has no replicas
   [junit4]   2> 64155 T134 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:56405/collection1/ shard1
   [junit4]   2> 64161 T100 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 64161 T100 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 64162 T100 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 64162 T100 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 64164 T141 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@45dc73ac name:ZooKeeperConnection Watcher:127.0.0.1:56393/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 64164 T100 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 64165 T100 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 64168 T126 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 64168 T138 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 64174 T134 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 64175 T127 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 64176 T127 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 64180 T126 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 64182 T127 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 64184 T126 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 64184 T138 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 64189 T127 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:56405",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 3
   [junit4]   2> 64195 T142 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 64197 T142 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 64197 T138 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 64233 T134 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:56405/collection1/ and leader is http://127.0.0.1:56405/collection1/
   [junit4]   2> 64234 T134 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:56405
   [junit4]   2> 64234 T134 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 64234 T134 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 64235 T134 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 64237 T142 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 64239 T127 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:56405",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:56405_",
   [junit4]   2> 	  "state":"active"} current state version: 4
   [junit4]   2> 64240 T127 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:56405",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:56405_",
   [junit4]   2> 	  "state":"active"}
   [junit4]   2> 64347 T138 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 64347 T142 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 65443 T100 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/shard-1-001/cores/collection1
   [junit4]   2> 65446 T100 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/shard-1-001
   [junit4]   2> 65447 T100 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 65449 T100 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:56426
   [junit4]   2> 65449 T100 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=56425, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/shard-1-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/tempDir-001/jetty1, hostContext=/, solrconfig=solrconfig.xml}
   [junit4]   2> 65450 T100 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 65450 T100 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/shard-1-001/'
   [junit4]   2> 65484 T100 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/shard-1-001/solr.xml
   [junit4]   2> 65528 T100 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/shard-1-001/cores
   [junit4]   2> 65529 T100 oasc.CoreContainer.<init> New CoreContainer 1157264384
   [junit4]   2> 65529 T100 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/shard-1-001/]
   [junit4]   2> 65530 T100 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/shard-1-001/lib
   [junit4]   2> 65531 T100 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/shard-1-001/lib).
   [junit4]   2> 65541 T100 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 65541 T100 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 65541 T100 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 65542 T100 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 65542 T100 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 65542 T100 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 65543 T100 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 65543 T100 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 65543 T100 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 65544 T100 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 65544 T100 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 65545 T100 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 65545 T100 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 65545 T100 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 65546 T100 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 65546 T100 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:56393/solr
   [junit4]   2> 65546 T100 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 65547 T100 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 65548 T100 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 65550 T154 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@472d792f name:ZooKeeperConnection Watcher:127.0.0.1:56393 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 65550 T100 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 65551 T100 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 65554 T100 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 65555 T157 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@109a40d4 name:ZooKeeperConnection Watcher:127.0.0.1:56393/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 65556 T100 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 65563 T100 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 66572 T100 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56425_
   [junit4]   2> 66573 T100 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:56425_
   [junit4]   2> 66578 T100 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 66589 T100 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/shard-1-001/cores
   [junit4]   2> 66591 T100 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, transient=false, name=collection1, schema=schema.xml, shard=, collection=collection1, dataDir=data/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/shard-1-001/cores/collection1, loadOnStartup=true, coreNodeName=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/shard-1-001/cores/collection1/}
   [junit4]   2> 66592 T100 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/shard-1-001/cores/collection1/
   [junit4]   2> 66592 T100 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 66596 T158 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 66596 T158 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 66599 T142 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 66601 T127 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:56425",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:56425_",
   [junit4]   2> 	  "state":"down"} current state version: 5
   [junit4]   2> 66601 T158 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 66601 T158 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 66601 T127 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:56425",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:56425_",
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 66602 T127 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 66602 T127 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 66602 T158 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 66604 T127 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 66611 T158 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 66611 T158 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 66612 T158 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 66612 T158 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 66613 T158 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 66613 T158 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 66614 T158 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 66614 T158 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/shard-1-001/cores/collection1/'
   [junit4]   2> 66628 T158 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 66632 T158 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 66646 T158 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 66668 T158 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 66716 T158 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 66817 T158 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 66840 T158 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 67257 T158 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 67278 T158 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 67281 T158 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 67307 T158 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 67313 T158 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 67318 T158 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 67320 T158 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 67320 T158 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 67320 T158 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 67322 T158 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 67322 T158 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 67322 T158 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 67323 T158 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 67323 T158 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 67323 T158 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@73f2f11
   [junit4]   2> 67325 T158 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/shard-1-001/cores/collection1/data
   [junit4]   2> 67325 T158 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 67325 T158 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 67325 T158 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 888997ABE0D7069C-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 67326 T158 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=13, maxMergeAtOnceExplicit=28, maxMergedSegmentMB=78.78125, floorSegmentMB=0.6962890625, forceMergeDeletesPctAllowed=6.197528975712406, segmentsPerTier=39.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 67327 T158 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@57ffa818 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5174ecca),segFN=segments_1,generation=1}
   [junit4]   2> 67327 T158 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 67338 T158 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 67339 T158 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 67339 T158 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 67340 T158 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 67340 T158 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 67340 T158 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 67341 T158 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 67341 T158 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 67341 T158 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 67365 T158 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 67369 T158 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 67372 T158 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 67375 T158 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 67406 T158 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/logging,/get,/update/csv,/admin/plugins,/admin/luke,standard,/admin/file,/update,/admin/threads,/admin/system,/admin/ping,/admin/segments,/admin/properties,/replication,/admin/mbeans,/update/json,/config,/schema,/update/json/docs
   [junit4]   2> 67408 T158 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 67410 T158 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 67410 T158 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 67412 T158 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 67412 T158 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 67414 T158 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=47, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6096064083982916]
   [junit4]   2> 67414 T158 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@57ffa818 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5174ecca),segFN=segments_1,generation=1}
   [junit4]   2> 67415 T158 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 67415 T158 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@db986d8[collection1] main
   [junit4]   2> 67415 T158 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 67417 T158 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 67417 T158 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 67418 T158 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 67418 T158 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 67418 T158 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 67419 T158 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 67420 T158 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 67420 T158 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 67420 T158 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 67423 T159 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@db986d8[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 67424 T158 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 67425 T162 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:56425 collection:collection1 shard:shard1
   [junit4]   2> 67425 T100 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 67426 T100 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 67426 T162 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 67433 T162 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 67436 T142 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 67436 T162 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 67437 T162 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C10 name=collection1 org.apache.solr.core.SolrCore@35879205 url=http://127.0.0.1:56425/collection1 node=127.0.0.1:56425_ C10_STATE=coll:collection1 core:collection1 props:{base_url=http://127.0.0.1:56425, core=collection1, node_name=127.0.0.1:56425_, state=down}
   [junit4]   2> 67437 T162 C:collection1 S:shard1 C10 P56425 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:56425/collection1/
   [junit4]   2> 67437 T162 C:collection1 S:shard1 C10 P56425 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 67438 T127 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 67438 T162 C:collection1 S:shard1 C10 P56425 oasc.SyncStrategy.syncToMe http://127.0.0.1:56425/collection1/ has no replicas
   [junit4]   2> 67438 T162 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:56425/collection1/ shard1
   [junit4]   2> 67438 T162 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 67440 T127 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 67441 T157 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 67442 T157 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 67447 T127 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:56425",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 67449 T127 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 67450 T157 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 67451 T157 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 67454 T142 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 67497 T162 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:56425/collection1/ and leader is http://127.0.0.1:56425/collection1/
   [junit4]   2> 67498 T162 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:56425
   [junit4]   2> 67498 T162 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 67498 T162 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 67499 T162 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 67501 T142 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 67502 T127 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection"

[...truncated too long message...]

	Maximum concurrent data watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		2	/solr/collections/repfacttest_c8n_1x3/leader_elect/shard1/election/93551136586596361-core_node3-n_0000000000
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		5	/solr/live_nodes
   [junit4]   2> 		4	/solr/overseer/queue
   [junit4]   2> 		4	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 758123 T2284 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 11 connections to: http://127.0.0.1:26423/nhff/w
   [junit4]   2> 758124 T2284 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 1 connections to: http://127.0.0.1:26398/nhff/w
   [junit4]   2> 758124 T2284 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 6 connections to: http://127.0.0.1:26413/nhff/w
   [junit4]   2> 758125 T2284 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:26444/nhff/w
   [junit4]   2> 758125 T2284 C:control_collection S:shard1 c:collection1 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 758125 T2284 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:26423/nhff/w
   [junit4]   2> 758126 T2284 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:26398/nhff/w
   [junit4]   2> 758126 T2284 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:26413/nhff/w
   [junit4]   2> 758126 T2284 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:26444/nhff/w
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=888997ABE0D7069C -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ar_TN -Dtests.timezone=Asia/Novokuznetsk -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   76.4s J3 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:26423/nhff/w/repfacttest_c8n_1x3_shard1_replica2
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([888997ABE0D7069C:DDA8714E2B6B64]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:26423/nhff/w/repfacttest_c8n_1x3_shard1_replica2
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 48 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 53 more
   [junit4]   2> 758145 T2284 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 888997ABE0D7069C-001
   [junit4]   2> 76441 T2283 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50), sim=RandomSimilarityProvider(queryNorm=false,coord=crazy): {}, locale=ar_TN, timezone=Asia/Novokuznetsk
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=118582752,total=276299776
   [junit4]   2> NOTE: All tests run in this JVM: [BasicZkTest, TestJsonRequest, DirectSolrConnectionTest, TestEmbeddedSolrServerConstructors, SimpleMLTQParserTest, TestImplicitCoreProperties, RollingRestartTest, TestReplicationHandler, SearchHandlerTest, TestDocSet, ZkSolrClientTest, ScriptEngineTest, TestBinaryResponseWriter, TestAtomicUpdateErrorCases, SuggesterTSTTest, TestSimpleQParserPlugin, FieldAnalysisRequestHandlerTest, BasicDistributedZk2Test, OpenExchangeRatesOrgProviderTest, RAMDirectoryFactoryTest, TestHashPartitioner, TestClusterStateMutator, TestTrie, MergeStrategyTest, TestIndexingPerformance, TestFaceting, TestWriterPerf, OutOfBoxZkACLAndCredentialsProvidersTest, TestCollationFieldDocValues, SSLMigrationTest, TestAnalyzeInfixSuggestions, TestManagedStopFilterFactory, SpellCheckCollatorTest, OverseerTest, TestExactStatsCache, TestInitQParser, DistanceUnitsTest, TestSolr4Spatial2, TestPerFieldSimilarity, TestCodecSupport, TestBadConfig, TestFastLRUCache, ReplicationFactorTest]
   [junit4] Completed on J3 in 77.47s, 1 test, 1 error <<< FAILURES!

[...truncated 888 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 62 ignored (22 assumptions)

Total time: 51 minutes 56 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 211100745 bytes
Compression is 0.0%
Took 37 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2843 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2843/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:36660/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:36660/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([99FF4379704481E0:11AB7CA3DEB8EC18]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([99FF4379704481E0:11AB7CA3DEB8EC18]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([99FF4379704481E0:11AB7CA3DEB8EC18]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9934 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/init-core-data-001
   [junit4]   2> 1146548 T2832 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 1146554 T2832 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1146555 T2833 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1146655 T2832 oasc.ZkTestServer.run start zk server on port:39638
   [junit4]   2> 1146656 T2832 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1146657 T2832 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1146666 T2840 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@68d7f7c name:ZooKeeperConnection Watcher:127.0.0.1:39638 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1146666 T2832 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1146667 T2832 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1146667 T2832 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1146671 T2832 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1146672 T2832 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1146673 T2843 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@12b49f97 name:ZooKeeperConnection Watcher:127.0.0.1:39638/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1146674 T2832 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1146674 T2832 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1146675 T2832 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1146678 T2832 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1146680 T2832 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1146682 T2832 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1146685 T2832 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1146686 T2832 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1146690 T2832 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1146691 T2832 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1146795 T2832 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1146795 T2832 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1146797 T2832 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1146798 T2832 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1146801 T2832 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1146802 T2832 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1146805 T2832 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1146806 T2832 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1146810 T2832 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1146811 T2832 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1146815 T2832 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1146816 T2832 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1146819 T2832 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1146821 T2832 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1146823 T2832 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1146824 T2832 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1146827 T2832 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1146828 T2832 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1151224 T2832 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/control-001/cores/collection1
   [junit4]   2> 1151231 T2832 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1151236 T2832 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:39643
   [junit4]   2> 1151236 T2832 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/tempDir-001/control/data, hostPort=39642}
   [junit4]   2> 1151237 T2832 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@749cd006
   [junit4]   2> 1151237 T2832 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/control-001/'
   [junit4]   2> 1151299 T2832 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/control-001/solr.xml
   [junit4]   2> 1151555 T2832 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/control-001/cores
   [junit4]   2> 1151556 T2832 oasc.CoreContainer.<init> New CoreContainer 539576863
   [junit4]   2> 1151556 T2832 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/control-001/]
   [junit4]   2> 1151556 T2832 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/control-001/lib
   [junit4]   2> 1151561 T2832 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/control-001/lib).
   [junit4]   2> 1151597 T2832 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1151598 T2832 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1151599 T2832 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1151599 T2832 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1151600 T2832 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1151600 T2832 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1151600 T2832 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1151601 T2832 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1151601 T2832 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1151602 T2832 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1151602 T2832 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1151603 T2832 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1151604 T2832 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1151604 T2832 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1151605 T2832 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1151605 T2832 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:39638/solr
   [junit4]   2> 1151606 T2832 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1151606 T2832 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1151607 T2832 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1151612 T2855 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@60396dcf name:ZooKeeperConnection Watcher:127.0.0.1:39638 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1151613 T2832 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1151614 T2832 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1151620 T2832 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1151623 T2858 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@11337bb9 name:ZooKeeperConnection Watcher:127.0.0.1:39638/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1151624 T2832 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1151627 T2832 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1151633 T2832 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1151640 T2832 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1151644 T2832 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1151648 T2832 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1151651 T2832 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1151653 T2832 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1151657 T2832 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1151660 T2832 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39642_
   [junit4]   2> 1151661 T2832 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:39642_
   [junit4]   2> 1151668 T2832 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1151672 T2832 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1151675 T2832 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1151679 T2832 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:39642_
   [junit4]   2> 1151680 T2832 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1151683 T2832 oasc.Overseer.start Overseer (id=93549997358055427-127.0.0.1:39642_-n_0000000000) starting
   [junit4]   2> 1151689 T2832 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1151704 T2832 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 1151706 T2860 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1151706 T2832 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1151706 T2859 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1151720 T2832 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/control-001/cores
   [junit4]   2> 1151731 T2832 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/control-001/cores/collection1/, name=collection1, dataDir=data/, loadOnStartup=true, transient=false, coreNodeName=, shard=, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/control-001/cores/collection1, collection=control_collection, config=solrconfig.xml}
   [junit4]   2> 1151732 T2832 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/control-001/cores/collection1/
   [junit4]   2> 1151732 T2832 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1151735 T2862 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1151735 T2862 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1151737 T2858 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1151737 T2862 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1151738 T2859 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:39642_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:39642"} current state version: 0
   [junit4]   2> 1151739 T2859 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:39642_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:39642"}
   [junit4]   2> 1151740 T2859 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 1151740 T2859 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1151743 T2858 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1152738 T2862 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1152739 T2862 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1152740 T2862 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1152740 T2862 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1152741 T2862 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1152741 T2862 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/control-001/cores/collection1/'
   [junit4]   2> 1152772 T2862 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1152779 T2862 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1152809 T2862 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1152864 T2862 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 1152958 T2862 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1153060 T2862 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1153071 T2862 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1153513 T2862 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1153529 T2862 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1153532 T2862 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1153542 T2862 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1153547 T2862 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1153552 T2862 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1153553 T2862 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1153553 T2862 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1153554 T2862 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1153554 T2862 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1153555 T2862 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1153555 T2862 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1153555 T2862 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1153555 T2862 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/control-001/cores/collection1/data/
   [junit4]   2> 1153556 T2862 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@42a2904a
   [junit4]   2> 1153557 T2862 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/control-001/cores/collection1/data
   [junit4]   2> 1153557 T2862 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/control-001/cores/collection1/data/index/
   [junit4]   2> 1153557 T2862 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1153557 T2862 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/control-001/cores/collection1/data/index
   [junit4]   2> 1153558 T2862 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=48, maxMergeAtOnceExplicit=24, maxMergedSegmentMB=13.7509765625, floorSegmentMB=2.080078125, forceMergeDeletesPctAllowed=17.303960195106118, segmentsPerTier=34.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7209390700854813
   [junit4]   2> 1153559 T2862 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5dd4aa77 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@35039ac1),segFN=segments_1,generation=1}
   [junit4]   2> 1153559 T2862 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1153564 T2862 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1153565 T2862 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1153565 T2862 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1153565 T2862 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1153566 T2862 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1153566 T2862 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1153566 T2862 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1153567 T2862 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1153567 T2862 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1153594 T2862 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1153596 T2862 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1153598 T2862 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1153601 T2862 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1153606 T2862 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update/json/docs,/admin/segments,standard,/replication,/admin/properties,/admin/system,/admin/logging,/update/json,/admin/plugins,/admin/file,/update,/admin/luke,/config,/schema,/admin/ping,/get,/admin/mbeans,/update/csv,/admin/threads
   [junit4]   2> 1153608 T2862 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1153609 T2862 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1153610 T2862 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1153611 T2862 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1153612 T2862 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1153612 T2862 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=940720259, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1153613 T2862 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5dd4aa77 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@35039ac1),segFN=segments_1,generation=1}
   [junit4]   2> 1153613 T2862 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1153614 T2862 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@3aab7661[collection1] main
   [junit4]   2> 1153614 T2862 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1153615 T2862 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1153615 T2862 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1153615 T2862 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1153615 T2862 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1153616 T2862 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1153616 T2862 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1153617 T2862 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1153617 T2862 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1153617 T2862 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1153618 T2863 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3aab7661[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1153618 T2862 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1153619 T2866 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:39642 collection:control_collection shard:shard1
   [junit4]   2> 1153619 T2832 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 1153620 T2832 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1153620 T2866 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1153621 T2832 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1153622 T2832 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1153623 T2870 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4da39c20 name:ZooKeeperConnection Watcher:127.0.0.1:39638/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1153623 T2832 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1153624 T2832 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1153624 T2832 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1153625 T2866 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1153627 T2858 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1153627 T2866 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1153627 T2866 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C449 name=collection1 org.apache.solr.core.SolrCore@46ee04fb url=http://127.0.0.1:39642/collection1 node=127.0.0.1:39642_ C449_STATE=coll:control_collection core:collection1 props:{core=collection1, node_name=127.0.0.1:39642_, state=down, base_url=http://127.0.0.1:39642}
   [junit4]   2> 1153628 T2866 C:control_collection S:shard1 C449 P39642 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:39642/collection1/
   [junit4]   2> 1153628 T2859 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 1153628 T2832 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1153628 T2866 C:control_collection S:shard1 C449 P39642 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1153628 T2832 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 1153629 T2866 C:control_collection S:shard1 C449 P39642 oasc.SyncStrategy.syncToMe http://127.0.0.1:39642/collection1/ has no replicas
   [junit4]   2> 1153629 T2832 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1153629 T2858 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1153629 T2870 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1153630 T2832 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1153629 T2866 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:39642/collection1/ shard1
   [junit4]   2> 1153631 T2866 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1153632 T2873 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@25346ac6 name:ZooKeeperConnection Watcher:127.0.0.1:39638/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1153632 T2832 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1153632 T2832 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1153634 T2858 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1153636 T2859 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 2, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 1153636 T2859 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 1153638 T2859 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:39642",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 1153638 T2859 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 1153639 T2858 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1153639 T2870 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1153642 T2858 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1153642 T2870 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1153642 T2858 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1153686 T2866 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:39642/collection1/ and leader is http://127.0.0.1:39642/collection1/
   [junit4]   2> 1153687 T2866 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:39642
   [junit4]   2> 1153687 T2866 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1153688 T2866 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1153688 T2866 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1153690 T2858 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1153691 T2859 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:39642_",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:39642"} current state version: 4
   [junit4]   2> 1153692 T2859 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:39642_",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:39642"}
   [junit4]   2> 1153797 T2858 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1153798 T2870 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1157567 T2832 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/shard-1-001/cores/collection1
   [junit4]   2> 1157576 T2832 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/shard-1-001
   [junit4]   2> 1157577 T2832 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1157582 T2832 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:20639
   [junit4]   2> 1157583 T2832 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=49478, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/tempDir-001/jetty1, hostContext=/, solrconfig=solrconfig.xml, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/shard-1-001/cores}
   [junit4]   2> 1157584 T2832 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@749cd006
   [junit4]   2> 1157585 T2832 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/shard-1-001/'
   [junit4]   2> 1157704 T2832 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/shard-1-001/solr.xml
   [junit4]   2> 1157865 T2832 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/shard-1-001/cores
   [junit4]   2> 1157866 T2832 oasc.CoreContainer.<init> New CoreContainer 1611834027
   [junit4]   2> 1157867 T2832 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/shard-1-001/]
   [junit4]   2> 1157867 T2832 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/shard-1-001/lib
   [junit4]   2> 1157870 T2832 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/shard-1-001/lib).
   [junit4]   2> 1157889 T2832 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1157890 T2832 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1157890 T2832 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1157891 T2832 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1157891 T2832 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1157891 T2832 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1157892 T2832 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1157892 T2832 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1157893 T2832 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1157893 T2832 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1157894 T2832 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1157895 T2832 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1157895 T2832 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1157896 T2832 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1157896 T2832 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1157897 T2832 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:39638/solr
   [junit4]   2> 1157897 T2832 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1157898 T2832 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1157899 T2832 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1157902 T2885 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@35a7e896 name:ZooKeeperConnection Watcher:127.0.0.1:39638 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1157903 T2832 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1157904 T2832 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1157909 T2832 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1157911 T2888 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2b01166f name:ZooKeeperConnection Watcher:127.0.0.1:39638/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1157912 T2832 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1157918 T2832 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1158927 T2832 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49478_
   [junit4]   2> 1158929 T2832 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49478_
   [junit4]   2> 1158934 T2832 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1158944 T2832 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/shard-1-001/cores
   [junit4]   2> 1158954 T2832 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/shard-1-001/cores/collection1/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/shard-1-001/cores/collection1, dataDir=data/, config=solrconfig.xml, name=collection1, transient=false, loadOnStartup=true, schema=schema.xml, coreNodeName=, shard=}
   [junit4]   2> 1158955 T2832 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/shard-1-001/cores/collection1/
   [junit4]   2> 1158956 T2832 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1158959 T2889 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1158960 T2889 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1158962 T2858 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1158963 T2889 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 1158963 T2889 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 1158963 T2859 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:49478_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49478"} current state version: 5
   [junit4]   2> 1158964 T2859 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:49478_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49478"}
   [junit4]   2> 1158965 T2889 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 1158965 T2859 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 1158965 T2859 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1158969 T2859 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 1158970 T2889 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 1158971 T2889 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1158971 T2888 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1158972 T2888 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 1159971 T2889 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1159972 T2889 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1159973 T2889 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1159973 T2889 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1159974 T2889 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1159974 T2889 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/shard-1-001/cores/collection1/'
   [junit4]   2> 1160014 T2889 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1160019 T2889 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1160055 T2889 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1160112 T2889 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 1160248 T2889 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1160351 T2889 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1160374 T2889 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1161222 T2889 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1161254 T2889 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1161258 T2889 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1161270 T2889 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1161275 T2889 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1161280 T2889 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1161281 T2889 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1161281 T2889 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1161281 T2889 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1161282 T2889 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1161282 T2889 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1161283 T2889 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1161283 T2889 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1161283 T2889 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 1161283 T2889 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@42a2904a
   [junit4]   2> 1161284 T2889 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/shard-1-001/cores/collection1/data
   [junit4]   2> 1161285 T2889 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 1161285 T2889 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1161285 T2889 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest 99FF4379704481E0-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 1161286 T2889 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=48, maxMergeAtOnceExplicit=24, maxMergedSegmentMB=13.7509765625, floorSegmentMB=2.080078125, forceMergeDeletesPctAllowed=17.303960195106118, segmentsPerTier=34.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7209390700854813
   [junit4]   2> 1161287 T2889 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@62cf2996 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@eb36988),segFN=segments_1,generation=1}
   [junit4]   2> 1161287 T2889 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1161294 T2889 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1161294 T2889 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1161295 T2889 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1161295 T2889 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1161295 T2889 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1161295 T2889 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1161296 T2889 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1161296 T2889 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1161296 T2889 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1161325 T2889 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1161328 T2889 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1161330 T2889 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1161333 T2889 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1161340 T2889 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update/json/docs,/admin/segments,standard,/replication,/admin/properties,/admin/system,/admin/logging,/update/json,/admin/plugins,/admin/file,/update,/admin/luke,/config,/schema,/admin/ping,/get,/admin/mbeans,/update/csv,/admin/threads
   [junit4]   2> 1161342 T2889 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1161344 T2889 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1161344 T2889 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1161346 T2889 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1161346 T2889 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1161348 T2889 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=940720259, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1161349 T2889 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@62cf2996 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@eb36988),segFN=segments_1,generation=1}
   [junit4]   2> 1161349 T2889 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1161349 T2889 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@4989fbf4[collection1] main
   [junit4]   2> 1161350 T2889 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1161351 T2889 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1161351 T2889 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1161351 T2889 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1161351 T2889 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1161352 T2889 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1161352 T2889 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1161353 T2889 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1161353 T2889 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1161353 T2889 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1161354 T2890 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4989fbf4[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1161355 T2889 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1161356 T2893 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:49478 collection:collection1 shard:shard1
   [junit4]   2> 1161356 T2832 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 1161356 T2832 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1161357 T2893 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1161361 T2893 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1161363 T2858 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1161363 T2893 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1161364 T2893 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C450 name=collection1 org.apache.solr.core.SolrCore@76568c76 url=http://127.0.0.1:49478/collection1 node=127.0.0.1:49478_ C450_STATE=coll:collection1 core:collection1 props:{core=collection1, node_name=127.0.0.1:49478_, state=down, base_url=http://127.0.0.1:49478}
   [junit4]   2> 1161364 T2893 C:collection1 S:shard1 C450 P49478 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:49478/collection1/
   [junit4]   2> 1161364 T2859 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 1161365 T2893 C:collection1 S:shard1 C450 P49478 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1161365 T2893 C:collection1 S:shard1 C450 P49478 oasc.SyncStrategy.syncToMe http://127.0.0.1:49478/collection1/ has no replicas
   [junit4]   2> 1161365 T2893 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:49478/collection1/ shard1
   [junit4]   2> 1161366 T2893 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1161366 T2859 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 1161367 T2888 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1161367 T2888 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 1161370 T2859 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49478",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 1161372 T2859 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 1161372 T2888 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1161373 T2888 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 1161375 T2858 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1161420 T2893 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:49478/collection1/ and leader is http://127.0.0.1:49478/collection1/
   [junit4]   2> 1161421 T2893 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:49478
   [junit4]   2> 1161421 T2893 C:collection1 S:shard1 oasc.ZkController.checkReco

[...truncated too long message...]

eue-work
   [junit4]   2> 		4	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1584518 T3425 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:36813/ob/jo
   [junit4]   2> 1584518 T3425 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 9 connections to: http://127.0.0.1:36805/ob/jo
   [junit4]   2> 1584519 T3425 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:36829/ob/jo
   [junit4]   2> 1584519 T3425 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 3 connections to: http://127.0.0.1:36818/ob/jo
   [junit4]   2> 1584519 T3425 C:control_collection S:shard1 c:collection1 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 1584520 T3425 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:36813/ob/jo
   [junit4]   2> 1584520 T3425 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:36805/ob/jo
   [junit4]   2> 1584520 T3425 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:36829/ob/jo
   [junit4]   2> 1584520 T3425 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:36818/ob/jo
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=99FF4379704481E0 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=sr_CS -Dtests.timezone=Europe/Jersey -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   81.3s J2 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([99FF4379704481E0:11AB7CA3DEB8EC18]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 1584548 T3425 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest 99FF4379704481E0-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {text=Lucene50(blocksize=128), intDefault=FST50, timestamp=FST50, id=PostingsFormat(name=Asserting), range_facet_l=FST50, _version_=FST50, multiDefault=PostingsFormat(name=Memory doPackFST= false), a_t=FST50}, docValues:{timestamp=DocValuesFormat(name=Memory)}, sim=DefaultSimilarity, locale=sr_CS, timezone=Europe/Jersey
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=100555688,total=299368448
   [junit4]   2> NOTE: All tests run in this JVM: [CurrencyFieldXmlFileTest, TestArbitraryIndexDir, TestIndexingPerformance, TestComponentsName, DistributedFacetPivotLongTailTest, BufferStoreTest, LeaderElectionTest, RollingRestartTest, SchemaVersionSpecificBehaviorTest, TestManagedSchemaFieldTypeResource, TestValueSourceCache, TestFaceting, HighlighterMaxOffsetTest, HighlighterTest, TestCoreDiscovery, FieldAnalysisRequestHandlerTest, TestCSVLoader, SimplePostToolTest, TestCloudPivotFacet, TestDistributedSearch, TestFieldCollectionResource, BasicDistributedZkTest, DisMaxRequestHandlerTest, DirectUpdateHandlerOptimizeTest, TestLMDirichletSimilarityFactory, NoCacheHeaderTest, TestCSVResponseWriter, TestManagedSynonymFilterFactory, TestWordDelimiterFilterFactory, UnloadDistributedZkTest, TestCustomSort, TestExpandComponent, TestRawResponseWriter, TestFastLRUCache, DistributedMLTComponentTest, DefaultValueUpdateProcessorTest, JSONWriterTest, CoreMergeIndexesAdminHandlerTest, TestSchemaManager, TestPHPSerializedResponseWriter, TestHdfsUpdateLog, IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestDFRSimilarityFactory, ZkCLITest, SpellCheckCollatorTest, DistributedExpandComponentTest, CoreAdminCreateDiscoverTest, TestMissingGroups, TestManagedSchema, TestJsonFacets, SystemInfoHandlerTest, DateRangeFieldTest, TestBulkSchemaAPI, DocValuesMissingTest, TestCryptoKeys, TestPhraseSuggestions, TestCodecSupport, TestNoOpRegenerator, TestReversedWildcardFilterFactory, SyncSliceTest, ChaosMonkeyNothingIsSafeTest, TestCloudSchemaless, ReplicationFactorTest]
   [junit4] Completed on J2 in 81.30s, 1 test, 1 error <<< FAILURES!

[...truncated 714 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 62 ignored (22 assumptions)

Total time: 53 minutes 3 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 222338042 bytes
Compression is 0.0%
Took 46 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2842 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2842/

4 tests failed.
REGRESSION:  org.apache.solr.handler.component.DistributedMLTComponentTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:51164//collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:51164//collection1
	at __randomizedtesting.SeedInfo.seed([A2D7B7BFA916A33E:2A83886507EACEC6]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:566)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:943)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:556)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:604)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:586)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:565)
	at org.apache.solr.handler.component.DistributedMLTComponentTest.test(DistributedMLTComponentTest.java:126)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 51 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:46512/qy/zb/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:46512/qy/zb/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([A2D7B7BFA916A33E:2A83886507EACEC6]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:11791/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:11791/collection1
	at __randomizedtesting.SeedInfo.seed([A2D7B7BFA916A33E:2A83886507EACEC6]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:11791/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:34693/w/repfacttest_c8n_1x3_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:34693/w/repfacttest_c8n_1x3_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([A2D7B7BFA916A33E:2A83886507EACEC6]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:34693/w/repfacttest_c8n_1x3_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9252 lines...]
   [junit4] Suite: org.apache.solr.handler.component.DistributedMLTComponentTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/init-core-data-001
   [junit4]   2> 23053 T43 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 23053 T43 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 25079 T43 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/control/cores/collection1
   [junit4]   2> 26051 T43 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 26629 T43 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:60854
   [junit4]   2> 26630 T43 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, hostPort=60854, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/control/cores}
   [junit4]   2> 26904 T43 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3972aa3f
   [junit4]   2> 26905 T43 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/control/'
   [junit4]   2> 26998 T43 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/control/solr.xml
   [junit4]   2> 27126 T43 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/control/cores
   [junit4]   2> 27127 T43 oasc.CoreContainer.<init> New CoreContainer 1719960860
   [junit4]   2> 27128 T43 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/control/]
   [junit4]   2> 27128 T43 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/control/lib
   [junit4]   2> 27130 T43 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/control/lib).
   [junit4]   2> 27263 T43 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 27263 T43 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 27264 T43 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 27264 T43 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 27265 T43 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 27265 T43 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 27266 T43 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 27266 T43 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 27267 T43 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 27267 T43 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 27268 T43 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 27269 T43 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 27269 T43 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 27270 T43 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 27270 T43 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 27272 T43 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/control/cores
   [junit4]   2> 27276 T43 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, configSet=collection1, transient=false, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/control/cores/collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/control/cores/collection1/, dataDir=data/, loadOnStartup=true, schema=schema.xml}
   [junit4]   2> 27277 T43 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/control/cores/collection1/
   [junit4]   2> 27277 T43 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 27279 T55 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 27281 T55 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 27282 T55 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 27282 T55 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 27463 T55 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 27573 T55 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 27680 T55 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 27681 T55 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 27777 T55 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 28465 T55 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 28513 T55 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 28516 T55 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 28567 T55 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 28573 T55 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 28578 T55 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 28585 T55 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 28585 T55 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 28586 T55 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 28587 T55 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 28587 T55 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 28604 T55 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from configset /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1
   [junit4]   2> 28604 T55 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 28605 T55 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/control/cores/collection1/data/
   [junit4]   2> 28606 T55 oasc.JmxMonitoredMap.<init> No JMX servers found, not exposing Solr information with JMX.
   [junit4]   2> 28607 T55 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/control/cores/collection1/data
   [junit4]   2> 28607 T55 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/control/cores/collection1/data/index/
   [junit4]   2> 28607 T55 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/control/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 28608 T55 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/control/cores/collection1/data/index
   [junit4]   2> 28608 T55 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=50, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=42.6044921875, floorSegmentMB=1.1875, forceMergeDeletesPctAllowed=20.71047839162438, segmentsPerTier=46.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.2522598961647321
   [junit4]   2> 28610 T55 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5205dd1e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7ef806a3),segFN=segments_1,generation=1}
   [junit4]   2> 28610 T55 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 28884 T55 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 29006 T55 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 29014 T55 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 29014 T55 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 29014 T55 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 29015 T55 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 29015 T55 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 29058 T55 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 29058 T55 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 29295 T55 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 29296 T55 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 29296 T55 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 29320 T55 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 29323 T55 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 29325 T55 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 29328 T55 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 29539 T55 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 29648 T55 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/plugins,tvrh,/config,/update,mltrh,/admin/properties,/admin/ping,/schema,/admin/luke,lazy,/search-facet-def,/admin/segments,/get,spellCheckCompRH_Direct,spellCheckCompRH1,dismax,standard,/replication,/mlt,/search-facet-invariants,/admin/file,defaults,spellCheckWithWordbreak,mock,/terms,/admin/threads,/admin/mbeans,spellCheckCompRH,/debug/dump,/admin/logging,/admin/system,spellCheckWithWordbreak_Direct,/update/json/docs,/update/csv,/update/json
   [junit4]   2> 29658 T55 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 29660 T55 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 29688 T55 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 29688 T55 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 29721 T55 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 29722 T55 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 29723 T55 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=20, maxMergeAtOnceExplicit=48, maxMergedSegmentMB=26.3857421875, floorSegmentMB=1.5673828125, forceMergeDeletesPctAllowed=14.079935294757192, segmentsPerTier=17.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 29724 T55 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5205dd1e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7ef806a3),segFN=segments_1,generation=1}
   [junit4]   2> 29724 T55 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 29725 T55 oass.SolrIndexSearcher.<init> Opening Searcher@6095cc5b[collection1] main
   [junit4]   2> 29726 T55 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 29726 T55 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 29726 T55 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 29727 T55 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 30174 T55 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 30776 T55 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 30777 T56 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 30780 T56 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 30781 T56 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 30781 T56 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 30784 T56 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 30787 T56 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 30787 T55 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 30788 T43 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 30788 T56 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 30789 T43 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 30790 T56 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 30792 T56 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 30792 T56 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6095cc5b[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 32815 T43 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard0/cores/collection1
   [junit4]   2> 32819 T43 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 32821 T43 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:15395
   [junit4]   2> 32822 T43 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard0/cores, hostPort=15395, hostContext=/}
   [junit4]   2> 32823 T43 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3972aa3f
   [junit4]   2> 32823 T43 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard0/'
   [junit4]   2> 32883 T43 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard0/solr.xml
   [junit4]   2> 32959 T43 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard0/cores
   [junit4]   2> 32959 T43 oasc.CoreContainer.<init> New CoreContainer 1371224564
   [junit4]   2> 32960 T43 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard0/]
   [junit4]   2> 32960 T43 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard0/lib
   [junit4]   2> 32962 T43 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard0/lib).
   [junit4]   2> 32976 T43 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 32977 T43 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 32977 T43 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 32978 T43 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 32978 T43 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 32979 T43 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 32979 T43 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 32980 T43 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 32980 T43 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 32981 T43 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 32981 T43 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 32982 T43 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 32982 T43 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 32983 T43 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 32983 T43 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 32985 T43 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard0/cores
   [junit4]   2> 32988 T43 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {loadOnStartup=true, name=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard0/cores/collection1/, dataDir=data/, config=solrconfig.xml, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard0/cores/collection1, configSet=collection1, transient=false}
   [junit4]   2> 32988 T43 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard0/cores/collection1/
   [junit4]   2> 32989 T43 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 32990 T67 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 32992 T67 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 32993 T67 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 32993 T67 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 33079 T67 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 33120 T67 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 33212 T67 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 33213 T67 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 33273 T67 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 33726 T67 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 33752 T67 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 33755 T67 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 33779 T67 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 33785 T67 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 33791 T67 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 33792 T67 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 33792 T67 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 33793 T67 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 33794 T67 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 33794 T67 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 33794 T67 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from configset /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1
   [junit4]   2> 33795 T67 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 33795 T67 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard0/cores/collection1/data/
   [junit4]   2> 33796 T67 oasc.JmxMonitoredMap.<init> No JMX servers found, not exposing Solr information with JMX.
   [junit4]   2> 33797 T67 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard0/cores/collection1/data
   [junit4]   2> 33797 T67 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard0/cores/collection1/data/index/
   [junit4]   2> 33797 T67 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard0/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 33798 T67 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard0/cores/collection1/data/index
   [junit4]   2> 33799 T67 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=50, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=42.6044921875, floorSegmentMB=1.1875, forceMergeDeletesPctAllowed=20.71047839162438, segmentsPerTier=46.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.2522598961647321
   [junit4]   2> 33800 T67 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3731a3c6 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1bf73879),segFN=segments_1,generation=1}
   [junit4]   2> 33800 T67 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 33804 T67 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 33805 T67 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 33805 T67 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 33806 T67 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 33806 T67 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 33807 T67 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 33807 T67 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 33808 T67 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 33809 T67 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 33809 T67 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 33810 T67 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 33810 T67 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 33830 T67 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 33834 T67 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 33836 T67 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 33838 T67 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 33842 T67 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 33844 T67 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/plugins,tvrh,/config,/update,mltrh,/admin/properties,/admin/ping,/schema,/admin/luke,lazy,/search-facet-def,/admin/segments,/get,spellCheckCompRH_Direct,spellCheckCompRH1,dismax,standard,/replication,/mlt,/search-facet-invariants,/admin/file,defaults,spellCheckWithWordbreak,mock,/terms,/admin/threads,/admin/mbeans,spellCheckCompRH,/debug/dump,/admin/logging,/admin/system,spellCheckWithWordbreak_Direct,/update/json/docs,/update/csv,/update/json
   [junit4]   2> 33847 T67 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 33850 T67 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 33853 T67 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 33853 T67 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 33855 T67 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 33856 T67 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 33856 T67 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=20, maxMergeAtOnceExplicit=48, maxMergedSegmentMB=26.3857421875, floorSegmentMB=1.5673828125, forceMergeDeletesPctAllowed=14.079935294757192, segmentsPerTier=17.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 33857 T67 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3731a3c6 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1bf73879),segFN=segments_1,generation=1}
   [junit4]   2> 33858 T67 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 33858 T67 oass.SolrIndexSearcher.<init> Opening Searcher@60c0bab6[collection1] main
   [junit4]   2> 33859 T67 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 33860 T67 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 33861 T67 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 33861 T67 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 33898 T67 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 33951 T67 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 33952 T68 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 33953 T67 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 33953 T43 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 33953 T68 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 33954 T43 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 33954 T68 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 33955 T68 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 33956 T68 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 33957 T68 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 33958 T68 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 33959 T68 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 33960 T68 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 33960 T68 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@60c0bab6[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 35323 T43 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard1/cores/collection1
   [junit4]   2> 35326 T43 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 35328 T43 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:63448
   [junit4]   2> 35329 T43 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=63448, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard1/cores, hostContext=/}
   [junit4]   2> 35329 T43 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3972aa3f
   [junit4]   2> 35330 T43 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard1/'
   [junit4]   2> 35372 T43 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard1/solr.xml
   [junit4]   2> 35420 T43 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard1/cores
   [junit4]   2> 35421 T43 oasc.CoreContainer.<init> New CoreContainer 1290048702
   [junit4]   2> 35421 T43 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard1/]
   [junit4]   2> 35422 T43 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard1/lib
   [junit4]   2> 35423 T43 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard1/lib).
   [junit4]   2> 35433 T43 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 35434 T43 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 35434 T43 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 35435 T43 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 35435 T43 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 35436 T43 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 35436 T43 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 35436 T43 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 35437 T43 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 35437 T43 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 35438 T43 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 35438 T43 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 35439 T43 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 35439 T43 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 35440 T43 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 35441 T43 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard1/cores
   [junit4]   2> 35443 T43 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard1/cores/collection1/, schema=schema.xml, configSet=collection1, loadOnStartup=true, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard1/cores/collection1, dataDir=data/, config=solrconfig.xml, name=collection1}
   [junit4]   2> 35444 T43 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard1/cores/collection1/
   [junit4]   2> 35444 T43 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 35446 T79 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 35448 T79 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 35449 T79 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 35450 T79 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 35581 T79 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 35648 T79 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 35776 T79 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 35778 T79 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 35853 T79 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 36443 T79 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 36478 T79 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 36482 T79 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 36525 T79 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 36533 T79 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 36540 T79 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 36541 T79 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 36542 T79 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 36542 T79 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 36543 T79 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 36544 T79 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 36544 T79 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from configset /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1
   [junit4]   2> 36545 T79 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 36546 T79 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard1/cores/collection1/data/
   [junit4]   2> 36546 T79 oasc.JmxMonitoredMap.<init> No JMX servers found, not exposing Solr information with JMX.
   [junit4]   2> 36547 T79 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard1/cores/collection1/data
   [junit4]   2> 36548 T79 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard1/cores/collection1/data/index/
   [junit4]   2> 36548 T79 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard1/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 36549 T79 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard1/cores/collection1/data/index
   [junit4]   2> 36550 T79 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=50, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=42.6044921875, floorSegmentMB=1.1875, forceMergeDeletesPctAllowed=20.71047839162438, segmentsPerTier=46.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.2522598961647321
   [junit4]   2> 36552 T79 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1f2a543b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@13fffd79),segFN=segments_1,generation=1}
   [junit4]   2> 36552 T79 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 36558 T79 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 36559 T79 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 36560 T79 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 36560 T79 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 36561 T79 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 36561 T79 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 36562 T79 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 36563 T79 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 36564 T79 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 36565 T79 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 36566 T79 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 36566 T79 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 36597 T79 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 36600 T79 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 36603 T79 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 36606 T79 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 36611 T79 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 36613 T79 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/plugins,tvrh,/config,/update,mltrh,/admin/properties,/admin/ping,/schema,/admin/luke,lazy,/search-facet-def,/admin/segments,/get,spellCheckCompRH_Direct,spellCheckCompRH1,dismax,standard,/replication,/mlt,/search-facet-invariants,/admin/file,defaults,spellCheckWithWordbreak,mock,/terms,/admin/threads,/admin/mbeans,spellCheckCompRH,/debug/dump,/admin/logging,/admin/system,spellCheckWithWordbreak_Direct,/update/json/docs,/update/csv,/update/json
   [junit4]   2> 36616 T79 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 36619 T79 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 36622 T79 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 36622 T79 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 36625 T79 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 36626 T79 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 36626 T79 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=20, maxMergeAtOnceExplicit=48, maxMergedSegmentMB=26.3857421875, floorSegmentMB=1.5673828125, forceMergeDeletesPctAllowed=14.079935294757192, segmentsPerTier=17.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 36628 T79 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1f2a543b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@13fffd79),segFN=segments_1,generation=1}
   [junit4]   2> 36628 T79 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 36629 T79 oass.SolrIndexSearcher.<init> Opening Searcher@717023f[collection1] main
   [junit4]   2> 36630 T79 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 36630 T79 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 36631 T79 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 36631 T79 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 36661 T79 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 36739 T79 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 36741 T80 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 36741 T79 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 36742 T43 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 36742 T80 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 36743 T43 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 36744 T80 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 36745 T80 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 36746 T80 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 36747 T80 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 36748 T80 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 36750 T80 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 36751 T80 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 36751 T80 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@717023f[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 39141 T43 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard2/cores/collection1
   [junit4]   2> 39145 T43 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 39149 T43 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:51164
   [junit4]   2> 39150 T43 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard2/cores, hostContext=/, hostPort=51164}
   [junit4]   2> 39151 T43 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3972aa3f
   [junit4]   2> 39151 T43 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard2/'
   [junit4]   2> 39205 T43 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard2/solr.xml
   [junit4]   2> 39289 T43 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard2/cores
   [junit4]   2> 39290 T43 oasc.CoreContainer.<init> New CoreContainer 1806659349
   [junit4]   2> 39291 T43 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard2/]
   [junit4]   2> 39291 T43 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard2/lib
   [junit4]   2> 39292 T43 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard2/lib).
   [junit4]   2> 39307 T43 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 39308 T43 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 39309 T43 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 39309 T43 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 39310 T43 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 39310 T43 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 39311 T43 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 39312 T43 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 39312 T43 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 39313 T43 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 39313 T43 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 39314 T43 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 39315 T43 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 39316 T43 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 39316 T43 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 39318 T43 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard2/cores
   [junit4]   2> 39321 T43 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, dataDir=data/, schema=schema.xml, transient=false, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard2/cores/collection1/, config=solrconfig.xml, configSet=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard2/cores/collection1, loadOnStartup=true}
   [junit4]   2> 39321 T43 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest A2D7B7BFA916A33E-001/tempDir-001/shard2/cores/collection1/
   [junit4]   2> 39322 T43 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 39324 T91 oasc.SolrResourceLoa

[...truncated too long message...]

LetterWord connecting to 127.0.0.1:46504 46504
   [junit4]   2> 1637675 T6273 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/aliases.json
   [junit4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 		3	/solr/collections/c8n_1x3_commits/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 		4	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1637676 T6272 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:46507/qy/zb
   [junit4]   2> 1637676 T6272 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 13 connections to: http://127.0.0.1:46512/qy/zb
   [junit4]   2> 1637676 T6272 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:46549/qy/zb
   [junit4]   2> 1637676 T6272 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 3 connections to: http://127.0.0.1:46526/qy/zb
   [junit4]   2> 1637677 T6272 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:46538/qy/zb
   [junit4]   2> 1637677 T6272 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:46507/qy/zb
   [junit4]   2> 1637677 T6272 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:46512/qy/zb
   [junit4]   2> 1637677 T6272 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:46549/qy/zb
   [junit4]   2> 1637678 T6272 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:46526/qy/zb
   [junit4]   2> 1637678 T6272 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:46538/qy/zb
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderInitiatedRecoveryOnCommitTest -Dtests.method=test -Dtests.seed=A2D7B7BFA916A33E -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ar_BH -Dtests.timezone=America/Merida -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   78.8s J3 | LeaderInitiatedRecoveryOnCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:46512/qy/zb/c8n_1x3_commits_shard1_replica3
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([A2D7B7BFA916A33E:2A83886507EACEC6]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 48 more
   [junit4]   2> 1637685 T6272 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest A2D7B7BFA916A33E-001
   [junit4]   2> 78853 T6271 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=false,coord=yes): {}, locale=ar_BH, timezone=America/Merida
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=181565016,total=465043456
   [junit4]   2> NOTE: All tests run in this JVM: [CursorMarkTest, MoreLikeThisHandlerTest, TestCloudManagedSchema, PrimitiveFieldTypeTest, TestSolrXml, TestLRUStatsCache, HdfsBasicDistributedZk2Test, TestBulkSchemaConcurrent, MBeansHandlerTest, MinimalSchemaTest, FileUtilsTest, DistributedFacetPivotLargeTest, SpellCheckComponentTest, TestZkChroot, DocValuesMissingTest, TestRecoveryHdfs, BasicDistributedZk2Test, XmlUpdateRequestHandlerTest, ResponseLogComponentTest, SolrIndexSplitterTest, CustomCollectionTest, HdfsDirectoryFactoryTest, TestCloudPivotFacet, SolrTestCaseJ4Test, TestBadConfig, TestCustomSort, DisMaxRequestHandlerTest, RecoveryAfterSoftCommitTest, TestCollectionAPI, TestFastOutputStream, TestCloudManagedSchemaConcurrent, DirectUpdateHandlerOptimizeTest, RegexBoostProcessorTest, LukeRequestHandlerTest, ClusterStateTest, TestLazyCores, DebugComponentTest, TestRawResponseWriter, TestReplicationHandler, SolrCmdDistributorTest, SimpleCollectionCreateDeleteTest, RequestHandlersTest, HdfsUnloadDistributedZkTest, SSLMigrationTest, DistributedFacetPivotSmallAdvancedTest, PreAnalyzedFieldTest, TestSolrCoreProperties, TestAnalyzedSuggestions, SuggesterTest, CopyFieldTest, DirectUpdateHandlerTest, QueryEqualityTest, AtomicUpdatesTest, ReplicationFactorTest, TestUniqueKeyFieldResource, TestNamedUpdateProcessors, TestCollationFieldDocValues, TestLFUCache, AlternateDirectoryTest, BasicDistributedZkTest, TestSolrConfigHandlerCloud, LeaderInitiatedRecoveryOnCommitTest]
   [junit4] Completed on J3 in 79.38s, 1 test, 1 error <<< FAILURES!

[...truncated 727 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 4 errors, 62 ignored (22 assumptions)

Total time: 55 minutes 22 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 205860759 bytes
Compression is 0.0%
Took 39 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2841 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2841/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:27158/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:27158/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([F7A90672CD0B846F:7FFD39A863F7E997]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19363/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19363/collection1
	at __randomizedtesting.SeedInfo.seed([F7A90672CD0B846F:7FFD39A863F7E997]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19363/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:45880/w/og/repfacttest_c8n_1x3_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:45880/w/og/repfacttest_c8n_1x3_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([F7A90672CD0B846F:7FFD39A863F7E997]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:45880/w/og/repfacttest_c8n_1x3_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9267 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/init-core-data-001
   [junit4]   2> 39790 T20 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 39838 T20 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 39847 T21 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 40148 T20 oasc.ZkTestServer.run start zk server on port:16833
   [junit4]   2> 40305 T20 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 40594 T20 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 40854 T28 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7aaba36d name:ZooKeeperConnection Watcher:127.0.0.1:16833 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 40856 T20 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 40872 T20 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 40894 T20 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 41126 T20 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 41128 T20 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 41131 T31 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7a1b3866 name:ZooKeeperConnection Watcher:127.0.0.1:16833/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 41131 T20 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 41132 T20 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 41168 T20 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 41174 T20 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 41180 T20 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 41184 T20 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 41190 T20 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 41192 T20 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 41201 T20 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 41202 T20 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 41208 T20 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 41209 T20 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 41215 T20 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 41216 T20 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 41221 T20 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 41222 T20 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 41228 T20 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 41229 T20 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 41234 T20 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 41235 T20 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 41240 T20 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 41241 T20 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 41245 T20 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 41246 T20 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 41250 T20 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 41251 T20 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 41255 T20 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 41256 T20 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 43180 T20 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/control-001/cores/collection1
   [junit4]   2> 44000 T20 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 44437 T20 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:62631
   [junit4]   2> 44438 T20 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/control-001/cores, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/tempDir-001/control/data, hostPort=33570}
   [junit4]   2> 44644 T20 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 44645 T20 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/control-001/'
   [junit4]   2> 44697 T20 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/control-001/solr.xml
   [junit4]   2> 44773 T20 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/control-001/cores
   [junit4]   2> 44774 T20 oasc.CoreContainer.<init> New CoreContainer 1677170905
   [junit4]   2> 44775 T20 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/control-001/]
   [junit4]   2> 44776 T20 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/control-001/lib
   [junit4]   2> 44777 T20 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/control-001/lib).
   [junit4]   2> 44904 T20 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 44905 T20 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 44906 T20 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 44907 T20 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 44908 T20 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 44908 T20 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 44909 T20 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 44910 T20 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 44910 T20 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 44911 T20 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 44912 T20 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 44913 T20 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 44914 T20 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 44915 T20 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 44916 T20 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 44917 T20 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:16833/solr
   [junit4]   2> 44989 T20 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 44990 T20 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 44991 T20 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 44995 T44 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5393c049 name:ZooKeeperConnection Watcher:127.0.0.1:16833 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 44996 T20 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 44997 T20 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 45030 T20 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 45032 T47 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@269bb42b name:ZooKeeperConnection Watcher:127.0.0.1:16833/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 45033 T20 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 45060 T20 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 45081 T20 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 45094 T20 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 45099 T20 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 45104 T20 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 45143 T20 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 45147 T20 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 45151 T20 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 45155 T20 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33570_
   [junit4]   2> 45166 T20 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:33570_
   [junit4]   2> 45171 T20 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 45175 T20 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 45178 T20 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 45208 T20 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:33570_
   [junit4]   2> 45209 T20 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 45212 T20 oasc.Overseer.start Overseer (id=93548039702118403-127.0.0.1:33570_-n_0000000000) starting
   [junit4]   2> 45238 T20 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 45279 T20 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 45309 T49 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 45309 T20 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 45340 T48 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 45350 T20 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/control-001/cores
   [junit4]   2> 45369 T20 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/control-001/cores/collection1/, schema=schema.xml, collection=control_collection, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/control-001/cores/collection1, config=solrconfig.xml, dataDir=data/, coreNodeName=, name=collection1, loadOnStartup=true, transient=false, shard=}
   [junit4]   2> 45370 T20 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/control-001/cores/collection1/
   [junit4]   2> 45371 T20 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 45379 T51 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 45380 T51 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 45400 T47 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 45400 T51 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 45406 T48 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:33570_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33570"} current state version: 0
   [junit4]   2> 45447 T48 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:33570_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33570"}
   [junit4]   2> 45452 T48 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 45489 T48 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 45505 T47 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 46402 T51 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 46402 T51 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 46404 T51 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 46404 T51 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 46407 T51 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 46408 T51 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/control-001/cores/collection1/'
   [junit4]   2> 46453 T51 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 46477 T51 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 46524 T51 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 46560 T51 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 46620 T51 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 46625 T51 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 46757 T51 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 47309 T51 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 47348 T51 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 47352 T51 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 47397 T51 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 47406 T51 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 47414 T51 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 47416 T51 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 47417 T51 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 47418 T51 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 47419 T51 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 47420 T51 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 47421 T51 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 47421 T51 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 47422 T51 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/control-001/cores/collection1/data/
   [junit4]   2> 47439 T51 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1f12007b
   [junit4]   2> 47664 T51 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/control-001/cores/collection1/data
   [junit4]   2> 47665 T51 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/control-001/cores/collection1/data/index/
   [junit4]   2> 47665 T51 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 47666 T51 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/control-001/cores/collection1/data/index
   [junit4]   2> 47668 T51 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=383137699, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 47680 T51 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7060b63 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1dd5942f),segFN=segments_1,generation=1}
   [junit4]   2> 47691 T51 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 47708 T51 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 48009 T51 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 48033 T51 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 48034 T51 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 48034 T51 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 48035 T51 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 48330 T51 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 48331 T51 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 48332 T51 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 48363 T51 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 48380 T51 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 48385 T51 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 48390 T51 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 48458 T51 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/properties,/update/json/docs,/update/json,/update/csv,/get,/admin/logging,/admin/mbeans,/replication,/admin/plugins,/update,standard,/admin/file,/admin/threads,/admin/luke,/admin/ping,/schema,/admin/segments,/admin/system,/config
   [junit4]   2> 48471 T51 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 48585 T51 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 48586 T51 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 48623 T51 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 48624 T51 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 48626 T51 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=10, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6201118083812693]
   [junit4]   2> 48628 T51 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7060b63 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1dd5942f),segFN=segments_1,generation=1}
   [junit4]   2> 48628 T51 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 48629 T51 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@4accabac[collection1] main
   [junit4]   2> 48630 T51 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 48633 T51 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 48633 T51 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 48658 T51 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 48659 T51 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 48660 T51 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 48661 T51 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 48662 T51 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 48663 T51 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 48738 T51 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 48743 T52 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4accabac[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 48755 T51 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 48758 T55 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:33570 collection:control_collection shard:shard1
   [junit4]   2> 48759 T20 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 48760 T20 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 48821 T55 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 48838 T55 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 48843 T47 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 48844 T55 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 48845 T55 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 48845 T48 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C2 name=collection1 org.apache.solr.core.SolrCore@73e84841 url=http://127.0.0.1:33570/collection1 node=127.0.0.1:33570_ C2_STATE=coll:control_collection core:collection1 props:{state=down, node_name=127.0.0.1:33570_, core=collection1, base_url=http://127.0.0.1:33570}
   [junit4]   2> 48846 T55 C:control_collection S:shard1 C2 P33570 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:33570/collection1/
   [junit4]   2> 48848 T55 C:control_collection S:shard1 C2 P33570 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 48848 T55 C:control_collection S:shard1 C2 P33570 oasc.SyncStrategy.syncToMe http://127.0.0.1:33570/collection1/ has no replicas
   [junit4]   2> 48849 T55 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:33570/collection1/ shard1
   [junit4]   2> 48867 T47 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 48886 T55 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 48930 T47 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 48933 T48 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33570",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 48955 T20 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 48957 T20 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 48959 T59 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@64791a53 name:ZooKeeperConnection Watcher:127.0.0.1:16833/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 48960 T20 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 48961 T20 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 48961 T20 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 48969 T20 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 48969 T20 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 48970 T20 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 48971 T20 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 48972 T62 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@f4a73e3 name:ZooKeeperConnection Watcher:127.0.0.1:16833/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 48973 T20 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 48974 T20 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 48977 T47 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 48980 T48 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 48981 T48 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 48983 T59 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 48984 T47 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 49035 T55 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:33570/collection1/ and leader is http://127.0.0.1:33570/collection1/
   [junit4]   2> 49050 T55 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:33570
   [junit4]   2> 49051 T55 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 49051 T55 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 49052 T55 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 49055 T47 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 49058 T48 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:33570_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33570",
   [junit4]   2> 	  "core_node_name":"core_node1"} current state version: 3
   [junit4]   2> 49060 T48 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:33570_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33570",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 49061 T48 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 49064 T47 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 49064 T59 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 49176 T59 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 49177 T47 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 51183 T20 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/shard-1-001/cores/collection1
   [junit4]   2> 51186 T20 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/shard-1-001
   [junit4]   2> 51187 T20 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 51190 T20 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:36269
   [junit4]   2> 51190 T20 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, hostPort=24877, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/shard-1-001/cores}
   [junit4]   2> 51191 T20 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 51192 T20 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/shard-1-001/'
   [junit4]   2> 51238 T20 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/shard-1-001/solr.xml
   [junit4]   2> 51287 T20 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/shard-1-001/cores
   [junit4]   2> 51287 T20 oasc.CoreContainer.<init> New CoreContainer 309851033
   [junit4]   2> 51288 T20 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/shard-1-001/]
   [junit4]   2> 51288 T20 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/shard-1-001/lib
   [junit4]   2> 51289 T20 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/shard-1-001/lib).
   [junit4]   2> 51302 T20 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 51302 T20 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 51303 T20 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 51303 T20 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 51304 T20 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 51304 T20 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 51304 T20 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 51305 T20 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 51305 T20 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 51306 T20 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 51306 T20 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 51307 T20 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 51307 T20 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 51308 T20 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 51308 T20 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 51308 T20 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:16833/solr
   [junit4]   2> 51309 T20 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 51309 T20 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 51310 T20 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 51313 T74 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7544b1bc name:ZooKeeperConnection Watcher:127.0.0.1:16833 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 51314 T20 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 51315 T20 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 51318 T20 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 51321 T77 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4e16230e name:ZooKeeperConnection Watcher:127.0.0.1:16833/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 51321 T20 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 51331 T20 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 52344 T20 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:24877_
   [junit4]   2> 52346 T20 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:24877_
   [junit4]   2> 52351 T20 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 52360 T20 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/shard-1-001/cores
   [junit4]   2> 52363 T20 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/shard-1-001/cores/collection1/, dataDir=data/, coreNodeName=, name=collection1, transient=false, config=solrconfig.xml, schema=schema.xml, loadOnStartup=true, collection=collection1, shard=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/shard-1-001/cores/collection1}
   [junit4]   2> 52363 T20 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/shard-1-001/cores/collection1/
   [junit4]   2> 52364 T20 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 52369 T78 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 52369 T78 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 52372 T47 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 52374 T78 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 52374 T48 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:24877_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:24877"} current state version: 5
   [junit4]   2> 52374 T78 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 52375 T48 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:24877_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:24877"}
   [junit4]   2> 52376 T78 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 52376 T48 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 52377 T48 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 52379 T48 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 52386 T78 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 52386 T78 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 52387 T78 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 52387 T78 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 52388 T78 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 52389 T78 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 52390 T78 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 52391 T78 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/shard-1-001/cores/collection1/'
   [junit4]   2> 52412 T78 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 52416 T78 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 52441 T78 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 52472 T78 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 52541 T78 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 52543 T78 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 52641 T78 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 53230 T78 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 53259 T78 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 53264 T78 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 53305 T78 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 53313 T78 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 53320 T78 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 53322 T78 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 53322 T78 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 53323 T78 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 53325 T78 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 53325 T78 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 53326 T78 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 53326 T78 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 53327 T78 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 53327 T78 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1f12007b
   [junit4]   2> 53330 T78 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/shard-1-001/cores/collection1/data
   [junit4]   2> 53330 T78 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 53331 T78 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 53331 T78 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F7A90672CD0B846F-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 53332 T78 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=383137699, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 53334 T78 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@260fc5e2 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55888bf3),segFN=segments_1,generation=1}
   [junit4]   2> 53334 T78 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 53353 T78 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 53354 T78 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 53355 T78 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 53355 T78 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 53356 T78 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 53356 T78 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 53357 T78 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 53358 T78 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 53358 T78 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 53380 T78 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 53385 T78 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 53389 T78 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 53393 T78 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 53435 T78 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/properties,/update/json/docs,/update/json,/update/csv,/get,/admin/logging,/admin/mbeans,/replication,/admin/plugins,/update,standard,/admin/file,/admin/threads,/admin/luke,/admin/ping,/schema,/admin/segments,/admin/system,/config
   [junit4]   2> 53439 T78 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 53441 T78 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 53442 T78 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 53444 T78 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 53444 T78 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 53447 T78 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=10, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6201118083812693]
   [junit4]   2> 53448 T78 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@260fc5e2 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55888bf3),segFN=segments_1,generation=1}
   [junit4]   2> 53448 T78 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 53449 T78 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@438f8baa[collection1] main
   [junit4]   2> 53450 T78 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 53452 T78 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 53452 T78 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 53454 T78 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 53454 T78 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 53455 T78 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 53456 T78 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 53457 T78 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 53457 T78 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 53458 T78 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 53461 T79 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@438f8baa[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 53462 T78 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 53463 T82 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:24877 collection:collection1 shard:shard1
   [junit4]   2> 53464 T20 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 53465 T82 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 53465 T20 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 53475 T82 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 53478 T47 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 53479 T82 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 53479 T82 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C3 name=collection1 org.apache.solr.core.SolrCore@21a9afe5 url=http://127.0.0.1:24877/collection1 node=127.0.0.1:24877_ C3_STATE=coll:collection1 core:collection1 props:{state=down, node_name=127.0.0.1:24877_, core=collection1, base_url=http://127.0.0.1:24877}
   [junit4]   2> 53479 T82 C:collection1 S:shard1 C3 P24877 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:24877/collection1/
   [junit4]   2> 53480 T82 C:collection1 S:shard1 C3 P24877 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 53480 T48 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 53480 T82 C:collection1 S:shard1 C3 P24877 oasc.SyncStrategy.syncToMe http://127.0.0.1:24877/collection1/ has no replicas
   [junit4]   2> 53481 T82 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:24877/collection1/ shard1
   [junit4]   2> 53481 T82 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 53483 T48 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 53484 T77 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 53486 T77 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 53492 T48 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:24877",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 53494 T48 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 53496 T77 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 53497 T77 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 53501 T47 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 53539 T82 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:24877/collection1/ and leader is http://127.0.0.1:24877/collection1/
   [junit4]   2> 53540 T82 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:24877
   [junit4]   2> 53541 T82 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 53541 T82 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 53541 T82 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 53544 T47 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 53546 T48 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:24877_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:24877",
   [junit4]   2> 	  "core_node_name":"core_node1"} current state version: 5
   [junit4]   2> 53548 T48 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [juni

[...truncated too long message...]

sh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> 1627701 T5601 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:19337 19337
   [junit4]   2> 1627975 T5602 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:19337 19337
   [junit4]   2> 1627977 T5602 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 		2	/solr/collections/collection1/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1627978 T5601 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:19377/
   [junit4]   2> 1627978 T5601 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:19343/
   [junit4]   2> 1627979 T5601 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 7 connections to: http://127.0.0.1:19363/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=F7A90672CD0B846F -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=sk -Dtests.timezone=Canada/Yukon -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   71.2s J3 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19363/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([F7A90672CD0B846F:7FFD39A863F7E997]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19363/collection1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 49 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 54 more
   [junit4]   2> 1627994 T5601 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F7A90672CD0B846F-001
   [junit4]   2> 71248 T5600 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=DefaultSimilarity, locale=sk, timezone=Canada/Yukon
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=205569096,total=395313152
   [junit4]   2> NOTE: All tests run in this JVM: [TestSolrXml, TestSolrDeletionPolicy2, BlockDirectoryTest, LeaderInitiatedRecoveryOnCommitTest, TestManagedSchemaFieldResource, ZkStateWriterTest, OverriddenZkACLAndCredentialsProvidersTest, TestReversedWildcardFilterFactory, TestSimpleTrackingShardHandler, RequiredFieldsTest, TestFileDictionaryLookup, HdfsRecoveryZkTest, ResourceLoaderTest, DocValuesMultiTest, TestValueSourceCache, TestFreeTextSuggestions, CopyFieldTest, TestReloadAndDeleteDocs, BasicZkTest, TestDynamicFieldResource, DistributedFacetPivotLongTailTest, TestAtomicUpdateErrorCases, TestFaceting, CoreAdminHandlerTest, HdfsChaosMonkeySafeLeaderTest, DateFieldTest, BasicDistributedZk2Test, SolrInfoMBeanTest, TestCloudManagedSchemaConcurrent, LukeRequestHandlerTest, CoreMergeIndexesAdminHandlerTest, TestFieldCollectionResource, TestRemoteStreaming, TestCloudManagedSchema, SoftAutoCommitTest, TestQuerySenderNoQuery, SearchHandlerTest, NumericFieldsTest, TestSolr4Spatial, ScriptEngineTest, TestFastLRUCache, HdfsDirectoryTest, TestHighlightDedupGrouping, TestHighFrequencyDictionaryFactory, AliasIntegrationTest, NotRequiredUniqueKeyTest, TestDistributedMissingSort, OverseerTest, JsonLoaderTest, PrimitiveFieldTypeTest, SolrIndexSplitterTest, TriLevelCompositeIdRoutingTest, SpatialHeatmapFacetsTest, TestReplicationHandlerBackup, SSLMigrationTest, TestSchemaResource, DOMUtilTest, ChaosMonkeyNothingIsSafeTest, CustomCollectionTest, CollectionsAPIAsyncDistributedZkTest, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J3 in 71.54s, 1 test, 1 error <<< FAILURES!

[...truncated 713 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 62 ignored (22 assumptions)

Total time: 56 minutes 36 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 213867258 bytes
Compression is 0.0%
Took 44 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2840 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2840/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:61578/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:61578/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([BA5EEDFE00E149E2:320AD224AE1D241A]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:38166/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:38166/collection1
	at __randomizedtesting.SeedInfo.seed([BA5EEDFE00E149E2:320AD224AE1D241A]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:38166/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:29590/d_utb/i/repfacttest_c8n_1x3_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:29590/d_utb/i/repfacttest_c8n_1x3_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([BA5EEDFE00E149E2:320AD224AE1D241A]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:29590/d_utb/i/repfacttest_c8n_1x3_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9520 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/init-core-data-001
   [junit4]   2> 339167 T486 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 339172 T486 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 339177 T487 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 339273 T486 oasc.ZkTestServer.run start zk server on port:38149
   [junit4]   2> 339274 T486 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 339274 T486 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 339278 T494 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4145c0a2 name:ZooKeeperConnection Watcher:127.0.0.1:38149 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 339278 T486 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 339279 T486 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 339279 T486 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 339282 T486 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 339282 T486 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 339284 T497 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@73396fe0 name:ZooKeeperConnection Watcher:127.0.0.1:38149/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 339284 T486 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 339284 T486 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 339284 T486 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 339287 T486 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 339289 T486 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 339291 T486 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 339293 T486 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 339293 T486 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 339296 T486 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 339297 T486 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 339300 T486 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 339300 T486 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 339302 T486 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 339303 T486 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 339305 T486 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 339305 T486 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 339307 T486 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 339308 T486 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 339310 T486 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 339310 T486 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 339312 T486 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 339313 T486 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 339315 T486 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 339315 T486 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 339318 T486 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 339318 T486 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 339320 T486 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 339321 T486 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 340406 T486 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/control-001/cores/collection1
   [junit4]   2> 340410 T486 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 340411 T486 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:38156
   [junit4]   2> 340412 T486 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/control-001/cores, hostPort=38155, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/tempDir-001/control/data, hostContext=/}
   [junit4]   2> 340412 T486 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 340412 T486 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/control-001/'
   [junit4]   2> 340450 T486 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/control-001/solr.xml
   [junit4]   2> 340496 T486 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/control-001/cores
   [junit4]   2> 340496 T486 oasc.CoreContainer.<init> New CoreContainer 160596532
   [junit4]   2> 340497 T486 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/control-001/]
   [junit4]   2> 340497 T486 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/control-001/lib
   [junit4]   2> 340497 T486 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/control-001/lib).
   [junit4]   2> 340510 T486 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 340510 T486 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 340510 T486 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 340510 T486 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 340511 T486 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 340511 T486 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 340511 T486 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 340511 T486 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 340512 T486 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 340512 T486 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 340512 T486 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 340513 T486 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 340513 T486 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 340513 T486 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 340513 T486 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 340514 T486 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:38149/solr
   [junit4]   2> 340514 T486 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 340514 T486 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 340518 T486 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 340519 T509 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@57aa64bc name:ZooKeeperConnection Watcher:127.0.0.1:38149 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 340520 T486 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 340520 T486 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 340523 T486 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 340524 T512 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6f77c770 name:ZooKeeperConnection Watcher:127.0.0.1:38149/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 340524 T486 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 340526 T486 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 340529 T486 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 340531 T486 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 340533 T486 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 340536 T486 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 340538 T486 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 340540 T486 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 340542 T486 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 340543 T486 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38155_
   [junit4]   2> 340544 T486 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:38155_
   [junit4]   2> 340546 T486 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 340548 T486 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 340549 T486 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 340551 T486 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:38155_
   [junit4]   2> 340551 T486 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 340553 T486 oasc.Overseer.start Overseer (id=93547107045736451-127.0.0.1:38155_-n_0000000000) starting
   [junit4]   2> 340555 T486 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 340562 T486 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 340563 T514 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 340563 T486 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 340563 T513 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 340567 T486 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/control-001/cores
   [junit4]   2> 340569 T486 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=control_collection, coreNodeName=, dataDir=data/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/control-001/cores/collection1, schema=schema.xml, loadOnStartup=true, transient=false, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/control-001/cores/collection1/, config=solrconfig.xml, name=collection1, shard=}
   [junit4]   2> 340570 T486 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/control-001/cores/collection1/
   [junit4]   2> 340570 T486 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 340571 T516 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 340571 T516 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 340572 T512 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 340572 T516 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 340573 T513 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38155",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:38155_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1"} current state version: 0
   [junit4]   2> 340574 T513 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38155",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:38155_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 340574 T513 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 340574 T513 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 340575 T512 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 341572 T516 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 341573 T516 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 341574 T516 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 341574 T516 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 341575 T516 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 341576 T516 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/control-001/cores/collection1/'
   [junit4]   2> 341595 T516 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 341598 T516 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 341612 T516 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 341640 T516 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 341702 T516 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 341704 T516 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 341720 T516 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 342185 T516 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 342205 T516 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 342208 T516 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 342230 T516 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 342235 T516 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 342240 T516 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 342241 T516 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 342242 T516 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 342242 T516 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 342243 T516 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 342243 T516 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 342244 T516 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 342244 T516 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 342244 T516 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/control-001/cores/collection1/data/
   [junit4]   2> 342245 T516 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6a75c5d3
   [junit4]   2> 342246 T516 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/control-001/cores/collection1/data
   [junit4]   2> 342246 T516 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/control-001/cores/collection1/data/index/
   [junit4]   2> 342246 T516 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 342246 T516 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/control-001/cores/collection1/data/index
   [junit4]   2> 342247 T516 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=35, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 342248 T516 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@43a98bea lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3020d25a),segFN=segments_1,generation=1}
   [junit4]   2> 342248 T516 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 342256 T516 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 342257 T516 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 342257 T516 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 342257 T516 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 342258 T516 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 342258 T516 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 342258 T516 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 342259 T516 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 342259 T516 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 342281 T516 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 342284 T516 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 342286 T516 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 342289 T516 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 342297 T516 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/segments,/update/json/docs,/admin/properties,/admin/plugins,/admin/file,/admin/system,/get,/admin/luke,/update/json,/config,standard,/update/csv,/schema,/replication,/admin/ping,/update,/admin/mbeans,/admin/threads,/admin/logging
   [junit4]   2> 342300 T516 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 342302 T516 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 342302 T516 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 342304 T516 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 342304 T516 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 342305 T516 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=863635351, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 342306 T516 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@43a98bea lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3020d25a),segFN=segments_1,generation=1}
   [junit4]   2> 342306 T516 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 342306 T516 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@7cd97260[collection1] main
   [junit4]   2> 342307 T516 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 342308 T516 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 342308 T516 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 342309 T516 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 342309 T516 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 342310 T516 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 342310 T516 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 342311 T516 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 342311 T516 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 342311 T516 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 342312 T517 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7cd97260[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 342313 T516 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 342314 T520 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:38155 collection:control_collection shard:shard1
   [junit4]   2> 342314 T486 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 342315 T486 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 342316 T520 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 342318 T486 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 342318 T486 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 342320 T524 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3b37ae7b name:ZooKeeperConnection Watcher:127.0.0.1:38149/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 342320 T486 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 342321 T486 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 342321 T486 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 342324 T520 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 342325 T486 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 342326 T512 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 342327 T520 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 342327 T520 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C230 name=collection1 org.apache.solr.core.SolrCore@3eac8cfc url=http://127.0.0.1:38155/collection1 node=127.0.0.1:38155_ C230_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:38155, state=down, node_name=127.0.0.1:38155_, core=collection1}
   [junit4]   2> 342327 T520 C:control_collection S:shard1 C230 P38155 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:38155/collection1/
   [junit4]   2> 342327 T520 C:control_collection S:shard1 C230 P38155 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 342328 T513 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 342328 T520 C:control_collection S:shard1 C230 P38155 oasc.SyncStrategy.syncToMe http://127.0.0.1:38155/collection1/ has no replicas
   [junit4]   2> 342328 T520 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:38155/collection1/ shard1
   [junit4]   2> 342328 T520 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 342330 T524 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 342330 T512 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 342337 T513 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38155",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 342340 T512 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 342341 T512 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 342341 T524 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 342387 T520 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:38155/collection1/ and leader is http://127.0.0.1:38155/collection1/
   [junit4]   2> 342388 T520 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:38155
   [junit4]   2> 342388 T520 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 342388 T520 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 342388 T520 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 342390 T512 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 342392 T513 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38155",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:38155_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1"} current state version: 3
   [junit4]   2> 342393 T513 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38155",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:38155_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 342498 T524 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 342498 T512 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 343547 T486 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/shard-1-001/cores/collection1
   [junit4]   2> 343549 T486 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/shard-1-001
   [junit4]   2> 343550 T486 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 343552 T486 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:38167
   [junit4]   2> 343552 T486 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, hostPort=38166, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/tempDir-001/jetty1, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/shard-1-001/cores, solrconfig=solrconfig.xml}
   [junit4]   2> 343553 T486 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 343553 T486 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/shard-1-001/'
   [junit4]   2> 343587 T486 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/shard-1-001/solr.xml
   [junit4]   2> 343629 T486 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/shard-1-001/cores
   [junit4]   2> 343629 T486 oasc.CoreContainer.<init> New CoreContainer 1659497537
   [junit4]   2> 343629 T486 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/shard-1-001/]
   [junit4]   2> 343630 T486 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/shard-1-001/lib
   [junit4]   2> 343630 T486 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/shard-1-001/lib).
   [junit4]   2> 343640 T486 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 343640 T486 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 343641 T486 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 343641 T486 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 343641 T486 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 343641 T486 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 343641 T486 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 343642 T486 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 343642 T486 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 343642 T486 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 343642 T486 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 343643 T486 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 343643 T486 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 343643 T486 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 343644 T486 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 343644 T486 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:38149/solr
   [junit4]   2> 343644 T486 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 343644 T486 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 343645 T486 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 343647 T536 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5d2e157c name:ZooKeeperConnection Watcher:127.0.0.1:38149 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 343647 T486 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 343648 T486 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 343650 T486 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 343651 T539 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@841de89 name:ZooKeeperConnection Watcher:127.0.0.1:38149/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 343651 T486 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 343657 T486 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 344663 T486 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38166_
   [junit4]   2> 344664 T486 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:38166_
   [junit4]   2> 344668 T486 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 344670 T486 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/shard-1-001/cores
   [junit4]   2> 344672 T486 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {dataDir=data/, collection=collection1, loadOnStartup=true, name=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/shard-1-001/cores/collection1/, shard=, coreNodeName=, schema=schema.xml, transient=false, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/shard-1-001/cores/collection1}
   [junit4]   2> 344673 T486 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/shard-1-001/cores/collection1/
   [junit4]   2> 344673 T486 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 344674 T540 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 344674 T540 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 344675 T512 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 344675 T540 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 344676 T513 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38166",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:38166_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1"} current state version: 4
   [junit4]   2> 344676 T513 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38166",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:38166_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 344677 T513 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 344677 T513 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 344678 T539 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 344678 T524 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 344678 T512 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 345676 T540 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 345677 T540 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 345678 T540 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 345678 T540 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 345679 T540 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 345679 T540 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/shard-1-001/cores/collection1/'
   [junit4]   2> 345693 T540 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 345695 T540 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 345705 T540 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 345723 T540 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 345763 T540 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 345764 T540 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 345787 T540 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 346129 T540 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 346143 T540 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 346145 T540 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 346160 T540 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 346164 T540 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 346168 T540 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 346169 T540 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 346169 T540 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 346169 T540 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 346170 T540 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 346170 T540 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 346170 T540 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 346171 T540 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 346171 T540 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 346171 T540 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6a75c5d3
   [junit4]   2> 346172 T540 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/shard-1-001/cores/collection1/data
   [junit4]   2> 346172 T540 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 346172 T540 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 346172 T540 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest BA5EEDFE00E149E2-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 346173 T540 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=35, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 346173 T540 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7ca52379 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3dbc1cdd),segFN=segments_1,generation=1}
   [junit4]   2> 346174 T540 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 346180 T540 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 346180 T540 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 346181 T540 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 346181 T540 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 346181 T540 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 346181 T540 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 346182 T540 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 346182 T540 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 346182 T540 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 346197 T540 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 346198 T540 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 346200 T540 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 346202 T540 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 346209 T540 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/segments,/update/json/docs,/admin/properties,/admin/plugins,/admin/file,/admin/system,/get,/admin/luke,/update/json,/config,standard,/update/csv,/schema,/replication,/admin/ping,/update,/admin/mbeans,/admin/threads,/admin/logging
   [junit4]   2> 346211 T540 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 346212 T540 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 346213 T540 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 346214 T540 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 346214 T540 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 346215 T540 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=863635351, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 346216 T540 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7ca52379 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3dbc1cdd),segFN=segments_1,generation=1}
   [junit4]   2> 346216 T540 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 346216 T540 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@1814879[collection1] main
   [junit4]   2> 346216 T540 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 346218 T540 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 346218 T540 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 346218 T540 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 346218 T540 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 346219 T540 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 346219 T540 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 346219 T540 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 346220 T540 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 346220 T540 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 346221 T541 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1814879[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 346221 T540 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 346222 T544 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:38166 collection:collection1 shard:shard1
   [junit4]   2> 346222 T486 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 346222 T486 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 346223 T544 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 346228 T544 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 346230 T512 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 346230 T544 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 346231 T544 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C231 name=collection1 org.apache.solr.core.SolrCore@14249cc5 url=http://127.0.0.1:38166/collection1 node=127.0.0.1:38166_ C231_STATE=coll:collection1 core:collection1 props:{base_url=http://127.0.0.1:38166, state=down, node_name=127.0.0.1:38166_, core=collection1}
   [junit4]   2> 346231 T544 C:collection1 S:shard1 C231 P38166 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:38166/collection1/
   [junit4]   2> 346231 T513 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 346231 T544 C:collection1 S:shard1 C231 P38166 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 346232 T544 C:collection1 S:shard1 C231 P38166 oasc.SyncStrategy.syncToMe http://127.0.0.1:38166/collection1/ has no replicas
   [junit4]   2> 346232 T544 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:38166/collection1/ shard1
   [junit4]   2> 346232 T544 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 346233 T512 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 346233 T539 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 346233 T524 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 346237 T512 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 346238 T513 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38166",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 346342 T539 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 346342 T524 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 346342 T512 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 346390 T544 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:38166/collection1/ and leader is http://127.0.0.1:38166/collection1/
   [junit4]   2> 346391 T544 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:38166
   [junit4]   2> 346391 T544 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 346391 T544 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 346391 T544 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 346393 T512 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 346394 T513 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38166",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:38166_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1"} current state version: 7
   [junit4]   2> 346394 T513 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38166",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:38166_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 346503 T524 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 346503 T539 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 346503 T512 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 347232 T486 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jen

[...truncated too long message...]

llection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:29600/d_utb/i
   [junit4]   2> 1590205 T3556 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 3 connections to: http://127.0.0.1:29585/d_utb/i
   [junit4]   2> 1590206 T3556 C:control_collection S:shard1 c:collection1 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 1590206 T3556 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:29590/d_utb/i
   [junit4]   2> 1590207 T3556 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:29575/d_utb/i
   [junit4]   2> 1590207 T3556 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:29600/d_utb/i
   [junit4]   2> 1590207 T3556 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:29585/d_utb/i
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=BA5EEDFE00E149E2 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ga -Dtests.timezone=America/Chihuahua -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   78.4s J1 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:29590/d_utb/i/repfacttest_c8n_1x3_shard1_replica3
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([BA5EEDFE00E149E2:320AD224AE1D241A]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:29590/d_utb/i/repfacttest_c8n_1x3_shard1_replica3
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 48 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 53 more
   [junit4]   2> 1590228 T3556 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest BA5EEDFE00E149E2-001
   [junit4]   2> 78398 T3555 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {id=FSTOrd50, a_t=PostingsFormat(name=Memory doPackFST= false), text=PostingsFormat(name=Asserting), multiDefault=PostingsFormat(name=Direct), range_facet_l=PostingsFormat(name=Memory doPackFST= false), timestamp=PostingsFormat(name=Memory doPackFST= false), intDefault=PostingsFormat(name=Memory doPackFST= false), _version_=PostingsFormat(name=Memory doPackFST= false)}, docValues:{timestamp=DocValuesFormat(name=Lucene50)}, sim=DefaultSimilarity, locale=ga, timezone=America/Chihuahua
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=45235968,total=325582848
   [junit4]   2> NOTE: All tests run in this JVM: [CoreAdminCreateDiscoverTest, LeaderElectionTest, TestSort, PreAnalyzedUpdateProcessorTest, TestIndexingPerformance, TestReRankQParserPlugin, TestRandomFaceting, SuggesterTest, TestUtils, ShardRoutingCustomTest, ShardRoutingTest, MultiTermTest, DistributedFacetPivotLargeTest, LeaderFailoverAfterPartitionTest, SuggestComponentTest, DistanceUnitsTest, FastVectorHighlighterTest, PathHierarchyTokenizerFactoryTest, UnloadDistributedZkTest, ExternalFileFieldSortTest, EchoParamsTest, SOLR749Test, ClusterStateTest, TestSchemaNameResource, SolrInfoMBeanTest, SolrCoreTest, HardAutoCommitTest, TestSolrJ, TestTrackingShardHandlerFactory, DateMathParserTest, JSONWriterTest, TestFastOutputStream, DocExpirationUpdateProcessorFactoryTest, TestOmitPositions, RollingRestartTest, DistribCursorPagingTest, TestSearchPerf, FileBasedSpellCheckerTest, TestRandomDVFaceting, StatsComponentTest, TestRawResponseWriter, SimplePostToolTest, DebugComponentTest, OverseerCollectionProcessorTest, ChaosMonkeySafeLeaderTest, ScriptEngineTest, TestComponentsName, TestNonDefinedSimilarityFactory, HdfsChaosMonkeySafeLeaderTest, BufferStoreTest, TestTrie, TestRequestStatusCollectionAPI, TestAnalyzedSuggestions, ActionThrottleTest, TestValueSourceCache, ResponseLogComponentTest, TestReloadAndDeleteDocs, TestBlendedInfixSuggestions, ChaosMonkeyNothingIsSafeTest, CollectionsAPIDistributedZkTest, ReplicationFactorTest]
   [junit4] Completed on J1 in 78.68s, 1 test, 1 error <<< FAILURES!

[...truncated 708 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1901 tests, 3 errors, 62 ignored (22 assumptions)

Total time: 53 minutes 12 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 220468453 bytes
Compression is 0.0%
Took 46 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2839 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2839/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:28724/if_smh/uu/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28724/if_smh/uu/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([E93FD50EAED7B0F3:616BEAD4002BDD0B]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33059/og_az/pl/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33059/og_az/pl/collection1
	at __randomizedtesting.SeedInfo.seed([E93FD50EAED7B0F3:616BEAD4002BDD0B]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33059/og_az/pl/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:50853/ik/hy/repfacttest_c8n_1x3_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:50853/ik/hy/repfacttest_c8n_1x3_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([E93FD50EAED7B0F3:616BEAD4002BDD0B]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:50853/ik/hy/repfacttest_c8n_1x3_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9271 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/init-core-data-001
   [junit4]   2> 31208 T21 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /if_smh/uu
   [junit4]   2> 31268 T21 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 31282 T22 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 31683 T21 oasc.ZkTestServer.run start zk server on port:19480
   [junit4]   2> 31867 T21 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 32152 T21 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 32374 T29 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2aace7a7 name:ZooKeeperConnection Watcher:127.0.0.1:19480 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 32375 T21 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 32385 T21 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 32402 T21 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 32575 T21 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 32577 T21 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 32580 T32 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@469524c6 name:ZooKeeperConnection Watcher:127.0.0.1:19480/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 32580 T21 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 32581 T21 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 32607 T21 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 32612 T21 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 32616 T21 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 32620 T21 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 32624 T21 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 32662 T21 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 32669 T21 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 32670 T21 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 32675 T21 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 32676 T21 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 32682 T21 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 32683 T21 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 32688 T21 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 32689 T21 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 32693 T21 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 32694 T21 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 32698 T21 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 32699 T21 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 32703 T21 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 32704 T21 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 32708 T21 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 32709 T21 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 32713 T21 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 32714 T21 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 32718 T21 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 32719 T21 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 34658 T21 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/control-001/cores/collection1
   [junit4]   2> 35664 T21 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 36112 T21 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:64874
   [junit4]   2> 36113 T21 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=43399, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/tempDir-001/control/data, hostContext=/if_smh/uu, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/control-001/cores}
   [junit4]   2> 36333 T21 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 36334 T21 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/control-001/'
   [junit4]   2> 36395 T21 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/control-001/solr.xml
   [junit4]   2> 36487 T21 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/control-001/cores
   [junit4]   2> 36488 T21 oasc.CoreContainer.<init> New CoreContainer 1968990718
   [junit4]   2> 36489 T21 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/control-001/]
   [junit4]   2> 36489 T21 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/control-001/lib
   [junit4]   2> 36490 T21 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/control-001/lib).
   [junit4]   2> 36619 T21 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 36620 T21 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 36620 T21 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 36621 T21 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 36621 T21 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 36622 T21 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 36622 T21 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 36623 T21 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 36624 T21 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 36624 T21 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 36625 T21 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 36626 T21 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 36627 T21 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 36627 T21 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 36628 T21 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 36629 T21 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:19480/solr
   [junit4]   2> 36733 T21 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 36734 T21 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 36735 T21 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 36738 T45 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@224e41b2 name:ZooKeeperConnection Watcher:127.0.0.1:19480 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 36739 T21 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 36740 T21 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 36771 T21 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 36773 T48 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2d300e51 name:ZooKeeperConnection Watcher:127.0.0.1:19480/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 36774 T21 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 36801 T21 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 36821 T21 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 36836 T21 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 36841 T21 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 36846 T21 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 36894 T21 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 36899 T21 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 36903 T21 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 36906 T21 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43399_if_smh%2Fuu
   [junit4]   2> 36920 T21 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:43399_if_smh%2Fuu
   [junit4]   2> 36926 T21 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 36930 T21 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 36934 T21 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 36971 T21 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:43399_if_smh%2Fuu
   [junit4]   2> 36972 T21 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 36976 T21 oasc.Overseer.start Overseer (id=93546150806290435-127.0.0.1:43399_if_smh%2Fuu-n_0000000000) starting
   [junit4]   2> 37014 T21 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 37061 T21 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 37098 T50 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 37098 T21 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 37138 T49 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 37150 T21 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/control-001/cores
   [junit4]   2> 37176 T21 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {coreNodeName=, shard=, schema=schema.xml, name=collection1, dataDir=data/, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/control-001/cores/collection1/, config=solrconfig.xml, collection=control_collection, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/control-001/cores/collection1, transient=false, loadOnStartup=true}
   [junit4]   2> 37177 T21 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/control-001/cores/collection1/
   [junit4]   2> 37178 T21 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 37187 T52 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 37187 T52 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 37208 T48 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 37209 T52 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 37219 T49 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:43399_if_smh%2Fuu",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:43399/if_smh/uu",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null} current state version: 0
   [junit4]   2> 37306 T49 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:43399_if_smh%2Fuu",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:43399/if_smh/uu",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 37318 T49 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 37392 T49 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 37421 T48 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 38210 T52 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 38210 T52 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 38212 T52 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 38212 T52 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 38215 T52 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 38215 T52 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/control-001/cores/collection1/'
   [junit4]   2> 38297 T52 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 38326 T52 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 38384 T52 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 38426 T52 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 38503 T52 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 38506 T52 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 38609 T52 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 39413 T52 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 39476 T52 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 39481 T52 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 39563 T52 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 39572 T52 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 39581 T52 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 39590 T52 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 39590 T52 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 39592 T52 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 39594 T52 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 39594 T52 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 39614 T52 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 39615 T52 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 39616 T52 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/control-001/cores/collection1/data/
   [junit4]   2> 39624 T52 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@64760424
   [junit4]   2> 39784 T52 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/control-001/cores/collection1/data
   [junit4]   2> 39784 T52 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/control-001/cores/collection1/data/index/
   [junit4]   2> 39785 T52 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 39786 T52 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/control-001/cores/collection1/data/index
   [junit4]   2> 39788 T52 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=525699029, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 39790 T52 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7ba3ce43 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3f3ad01f),segFN=segments_1,generation=1}
   [junit4]   2> 39790 T52 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 39806 T52 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 39917 T52 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 39918 T52 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 39918 T52 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 39919 T52 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 39920 T52 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 39920 T52 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 39921 T52 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 39922 T52 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 39954 T52 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 39970 T52 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 39975 T52 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 39979 T52 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 40002 T52 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/mbeans,/admin/luke,/update/json,/admin/system,/admin/ping,/admin/segments,/config,/update,/update/csv,/replication,/get,/update/json/docs,/admin/file,/schema,standard,/admin/logging,/admin/threads,/admin/plugins,/admin/properties
   [junit4]   2> 40020 T52 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 40052 T52 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 40053 T52 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 40080 T52 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 40081 T52 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 40084 T52 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=32, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 40086 T52 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7ba3ce43 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3f3ad01f),segFN=segments_1,generation=1}
   [junit4]   2> 40087 T52 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 40089 T52 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@73690a3e[collection1] main
   [junit4]   2> 40090 T52 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 40093 T52 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 40094 T52 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 40115 T52 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 40116 T52 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 40117 T52 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 40118 T52 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 40119 T52 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 40120 T52 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 40121 T52 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 40124 T53 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@73690a3e[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 40137 T52 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 40139 T56 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:43399/if_smh/uu collection:control_collection shard:shard1
   [junit4]   2> 40139 T21 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 40140 T21 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 40182 T56 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 40192 T56 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 40195 T48 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 40195 T56 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 40196 T56 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C2 name=collection1 org.apache.solr.core.SolrCore@31fa23b2 url=http://127.0.0.1:43399/if_smh/uu/collection1 node=127.0.0.1:43399_if_smh%2Fuu C2_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:43399_if_smh%2Fuu, base_url=http://127.0.0.1:43399/if_smh/uu, state=down, core=collection1}
   [junit4]   2> 40196 T56 C:control_collection S:shard1 C2 P43399 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:43399/if_smh/uu/collection1/
   [junit4]   2> 40197 T49 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 40197 T56 C:control_collection S:shard1 C2 P43399 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 40198 T56 C:control_collection S:shard1 C2 P43399 oasc.SyncStrategy.syncToMe http://127.0.0.1:43399/if_smh/uu/collection1/ has no replicas
   [junit4]   2> 40199 T56 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:43399/if_smh/uu/collection1/ shard1
   [junit4]   2> 40211 T48 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 40216 T56 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 40223 T48 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 40225 T49 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:43399/if_smh/uu",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 40262 T21 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 40263 T21 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 40265 T60 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@32e1e333 name:ZooKeeperConnection Watcher:127.0.0.1:19480/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 40266 T21 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 40266 T21 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 40267 T21 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 40273 T21 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 40274 T21 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 40274 T21 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 40275 T21 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 40277 T63 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@533a78d7 name:ZooKeeperConnection Watcher:127.0.0.1:19480/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 40277 T21 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 40278 T21 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 40281 T48 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 40284 T49 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 40284 T49 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 40286 T48 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 40286 T60 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 40326 T56 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:43399/if_smh/uu/collection1/ and leader is http://127.0.0.1:43399/if_smh/uu/collection1/
   [junit4]   2> 40336 T56 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:43399/if_smh/uu
   [junit4]   2> 40336 T56 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 40337 T56 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 40337 T56 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 40340 T48 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 40343 T49 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:43399_if_smh%2Fuu",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:43399/if_smh/uu",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1"} current state version: 3
   [junit4]   2> 40344 T49 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:43399_if_smh%2Fuu",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:43399/if_smh/uu",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1"}
   [junit4]   2> 40345 T49 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 40348 T48 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 40348 T60 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 40460 T60 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 40460 T48 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 42340 T21 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/shard-1-001/cores/collection1
   [junit4]   2> 42344 T21 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/shard-1-001
   [junit4]   2> 42345 T21 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 42348 T21 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:56753
   [junit4]   2> 42349 T21 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/if_smh/uu, hostPort=30280, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/shard-1-001/cores}
   [junit4]   2> 42350 T21 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 42350 T21 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/shard-1-001/'
   [junit4]   2> 42403 T21 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/shard-1-001/solr.xml
   [junit4]   2> 42468 T21 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/shard-1-001/cores
   [junit4]   2> 42469 T21 oasc.CoreContainer.<init> New CoreContainer 83891931
   [junit4]   2> 42469 T21 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/shard-1-001/]
   [junit4]   2> 42469 T21 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/shard-1-001/lib
   [junit4]   2> 42470 T21 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/shard-1-001/lib).
   [junit4]   2> 42483 T21 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 42483 T21 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 42484 T21 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 42484 T21 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 42484 T21 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 42485 T21 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 42485 T21 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 42485 T21 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 42486 T21 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 42486 T21 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 42487 T21 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 42487 T21 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 42488 T21 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 42488 T21 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 42489 T21 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 42489 T21 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:19480/solr
   [junit4]   2> 42490 T21 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 42490 T21 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 42491 T21 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 42493 T75 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@9db3b5a name:ZooKeeperConnection Watcher:127.0.0.1:19480 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 42494 T21 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 42495 T21 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 42497 T21 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 42500 T78 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7dd63dc2 name:ZooKeeperConnection Watcher:127.0.0.1:19480/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 42500 T21 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 42552 T21 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 43563 T21 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:30280_if_smh%2Fuu
   [junit4]   2> 43565 T21 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:30280_if_smh%2Fuu
   [junit4]   2> 43569 T21 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 43581 T21 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/shard-1-001/cores
   [junit4]   2> 43584 T21 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {coreNodeName=, shard=, schema=schema.xml, collection=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/shard-1-001/cores/collection1/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/shard-1-001/cores/collection1, name=collection1, transient=false, config=solrconfig.xml, dataDir=data/, loadOnStartup=true}
   [junit4]   2> 43585 T21 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/shard-1-001/cores/collection1/
   [junit4]   2> 43585 T21 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 43590 T79 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 43590 T79 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 43594 T48 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 43595 T79 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 43596 T79 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 43596 T49 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:30280_if_smh%2Fuu",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:30280/if_smh/uu",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null} current state version: 5
   [junit4]   2> 43597 T79 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 43597 T49 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:30280_if_smh%2Fuu",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:30280/if_smh/uu",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 43598 T49 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 43598 T49 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 43601 T49 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 43606 T79 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 43606 T79 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 43607 T79 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 43607 T79 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 43608 T79 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 43609 T79 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 43610 T79 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 43611 T79 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/shard-1-001/cores/collection1/'
   [junit4]   2> 43633 T79 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 43637 T79 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 43656 T79 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 43680 T79 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 43728 T79 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 43731 T79 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 43786 T79 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 44263 T79 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 44288 T79 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 44291 T79 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 44328 T79 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 44336 T79 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 44342 T79 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 44344 T79 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 44345 T79 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 44345 T79 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 44346 T79 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 44347 T79 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 44347 T79 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 44348 T79 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 44348 T79 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 44348 T79 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@64760424
   [junit4]   2> 44350 T79 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/shard-1-001/cores/collection1/data
   [junit4]   2> 44351 T79 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 44351 T79 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 44351 T79 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest E93FD50EAED7B0F3-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 44352 T79 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=525699029, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 44353 T79 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@392a0078 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7484e399),segFN=segments_1,generation=1}
   [junit4]   2> 44354 T79 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 44366 T79 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 44366 T79 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 44367 T79 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 44367 T79 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 44368 T79 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 44368 T79 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 44369 T79 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 44370 T79 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 44370 T79 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 44394 T79 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 44399 T79 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 44402 T79 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 44406 T79 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 44441 T79 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/mbeans,/admin/luke,/update/json,/admin/system,/admin/ping,/admin/segments,/config,/update,/update/csv,/replication,/get,/update/json/docs,/admin/file,/schema,standard,/admin/logging,/admin/threads,/admin/plugins,/admin/properties
   [junit4]   2> 44444 T79 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 44446 T79 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 44446 T79 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 44448 T79 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 44449 T79 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 44451 T79 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=32, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 44452 T79 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@392a0078 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7484e399),segFN=segments_1,generation=1}
   [junit4]   2> 44452 T79 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 44452 T79 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@21a59e9b[collection1] main
   [junit4]   2> 44453 T79 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 44455 T79 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 44455 T79 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 44456 T79 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 44456 T79 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 44456 T79 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 44457 T79 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 44458 T79 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 44458 T79 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 44458 T79 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 44461 T80 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@21a59e9b[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 44462 T79 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 44464 T83 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:30280/if_smh/uu collection:collection1 shard:shard1
   [junit4]   2> 44464 T21 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 44465 T21 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 44465 T83 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 44474 T83 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 44478 T48 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 44478 T83 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 44479 T83 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C3 name=collection1 org.apache.solr.core.SolrCore@21a67480 url=http://127.0.0.1:30280/if_smh/uu/collection1 node=127.0.0.1:30280_if_smh%2Fuu C3_STATE=coll:collection1 core:collection1 props:{node_name=127.0.0.1:30280_if_smh%2Fuu, base_url=http://127.0.0.1:30280/if_smh/uu, state=down, core=collection1}
   [junit4]   2> 44479 T83 C:collection1 S:shard1 C3 P30280 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:30280/if_smh/uu/collection1/
   [junit4]   2> 44480 T49 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 44480 T83 C:collection1 S:shard1 C3 P30280 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 44481 T83 C:collection1 S:shard1 C3 P30280 oasc.SyncStrategy.syncToMe http://127.0.0.1:30280/if_smh/uu/collection1/ has no replicas
   [junit4]   2> 44481 T83 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:30280/if_smh/uu/collection1/ shard1
   [junit4]   2> 44482 T83 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 44483 T49 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 44484 T78 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 44486 T78 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 44492 T49 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:30280/if_smh/uu",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 44496 T49 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 44497 T78 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 44499 T78 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 44503 T48 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 44543 T83 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:30280/if_smh/uu/collection1/ and leader is http://127.0.0.1:30280/if_smh/uu/collection1/
   [junit4]   2> 44544 T83 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:30280/if_smh/uu
   [junit4]   2> 44544 T83 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 44545 T83 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 44545 T83 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 44548 T48 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 44550 T49 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:30280_if_smh%2Fuu",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1

[...truncated too long message...]

cting to 127.0.0.1:26479 26479
   [junit4]   2> 1580863 T4037 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 		2	/solr/collections/collection1/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1580863 T4036 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:33059/og_az/pl
   [junit4]   2> 1580863 T4036 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:57503/og_az/pl
   [junit4]   2> 1580863 T4036 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:65480/og_az/pl
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=E93FD50EAED7B0F3 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=lv_LV -Dtests.timezone=Pacific/Rarotonga -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   71.6s J1 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33059/og_az/pl/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([E93FD50EAED7B0F3:616BEAD4002BDD0B]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33059/og_az/pl/collection1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 49 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 54 more
   [junit4]   2> 1580895 T4036 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest E93FD50EAED7B0F3-001
   [junit4]   2> NOTE: test params are: codec=FastCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST, chunkSize=23213, maxDocsPerChunk=778, blockSize=44), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST, chunkSize=23213, blockSize=44)), sim=RandomSimilarityProvider(queryNorm=false,coord=crazy): {}, locale=lv_LV, timezone=Pacific/Rarotonga
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=225301432,total=300417024
   [junit4]   2> NOTE: All tests run in this JVM: [TestDocSet, TestSolrDynamicMBean, TestReversedWildcardFilterFactory, TestSolrJ, TestDefaultSimilarityFactory, CloudExitableDirectoryReaderTest, TestAddFieldRealTimeGet, TestDistributedMissingSort, DateMathParserTest, TestCSVLoader, NumericFieldsTest, TestSolrDeletionPolicy1, DistributedFacetPivotLargeTest, TestRTGBase, TestRemoteStreaming, TestPhraseSuggestions, IndexSchemaRuntimeFieldTest, TestManagedSchema, UpdateRequestProcessorFactoryTest, BasicZkTest, TestValueSourceCache, TestManagedResource, TestQuerySenderNoQuery, TestShardHandlerFactory, BadIndexSchemaTest, TestConfigOverlay, TestLFUCache, TestMacros, LeaderFailoverAfterPartitionTest, TestReloadAndDeleteDocs, TestArbitraryIndexDir, FileUtilsTest, OutputWriterTest, TestHdfsUpdateLog, DistributedMLTComponentTest, DistributedIntervalFacetingTest, StressHdfsTest, TestRebalanceLeaders, MoreLikeThisHandlerTest, CollectionsAPIDistributedZkTest, TestSearchPerf, TestDistributedGrouping, TestPseudoReturnFields, TestIndexSearcher, TestPerFieldSimilarity, FullSolrCloudDistribCmdsTest, ResponseHeaderTest, WordBreakSolrSpellCheckerTest, ExternalCollectionsTest, LukeRequestHandlerTest, TestManagedSynonymFilterFactory, TestStressLucene, DocumentAnalysisRequestHandlerTest, SoftAutoCommitTest, TestNoOpRegenerator, TimeZoneUtilsTest, HighlighterConfigTest, FieldMutatingUpdateProcessorTest, TestSolrConfigHandler, OutOfBoxZkACLAndCredentialsProvidersTest, TestFieldTypeResource, UnloadDistributedZkTest, CollectionsAPIAsyncDistributedZkTest, TestCloudManagedSchema, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J1 in 71.64s, 1 test, 1 error <<< FAILURES!

[...truncated 746 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1901 tests, 3 errors, 62 ignored (22 assumptions)

Total time: 54 minutes 27 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 218295612 bytes
Compression is 0.0%
Took 42 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2838 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2838/

4 tests failed.
REGRESSION:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test

Error Message:
shard1 is not consistent.  Got 134 from http://127.0.0.1:18704/ff/v/collection1lastClient and got 118 from http://127.0.0.1:18724/ff/v/collection1

Stack Trace:
java.lang.AssertionError: shard1 is not consistent.  Got 134 from http://127.0.0.1:18704/ff/v/collection1lastClient and got 118 from http://127.0.0.1:18724/ff/v/collection1
	at __randomizedtesting.SeedInfo.seed([6DEA3E9C667F7A53:E5BE0146C88317AB]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1310)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1289)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:161)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:53353/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:53353/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([6DEA3E9C667F7A53:E5BE0146C88317AB]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([6DEA3E9C667F7A53:E5BE0146C88317AB]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([6DEA3E9C667F7A53:E5BE0146C88317AB]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9465 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/init-core-data-001
   [junit4]   2> 285489 T891 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 285495 T891 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 285496 T892 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 285595 T891 oasc.ZkTestServer.run start zk server on port:48440
   [junit4]   2> 285596 T891 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 285596 T891 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 285599 T899 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5f47be4e name:ZooKeeperConnection Watcher:127.0.0.1:48440 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 285599 T891 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 285600 T891 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 285600 T891 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 285602 T891 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 285603 T891 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 285604 T902 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@206ceacc name:ZooKeeperConnection Watcher:127.0.0.1:48440/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 285604 T891 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 285605 T891 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 285605 T891 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 285607 T891 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 285608 T891 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 285609 T891 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 285610 T891 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 285611 T891 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 285613 T891 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 285613 T891 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 285615 T891 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 285615 T891 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 285616 T891 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 285617 T891 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 285618 T891 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 285618 T891 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 285620 T891 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 285620 T891 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 285621 T891 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 285622 T891 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 285623 T891 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 285623 T891 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 285625 T891 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 285625 T891 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 285626 T891 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 285627 T891 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 285628 T891 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 285628 T891 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 286670 T891 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/control-001/cores/collection1
   [junit4]   2> 286674 T891 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 286682 T891 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:21570
   [junit4]   2> 286682 T891 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/tempDir-001/control/data, hostPort=25938, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/control-001/cores, hostContext=/}
   [junit4]   2> 286683 T891 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 286683 T891 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/control-001/'
   [junit4]   2> 286725 T891 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/control-001/solr.xml
   [junit4]   2> 286780 T891 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/control-001/cores
   [junit4]   2> 286780 T891 oasc.CoreContainer.<init> New CoreContainer 282945584
   [junit4]   2> 286780 T891 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/control-001/]
   [junit4]   2> 286780 T891 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/control-001/lib
   [junit4]   2> 286781 T891 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/control-001/lib).
   [junit4]   2> 286791 T891 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 286791 T891 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 286791 T891 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 286791 T891 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 286792 T891 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 286792 T891 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 286792 T891 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 286792 T891 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 286792 T891 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 286793 T891 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 286793 T891 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 286793 T891 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 286794 T891 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 286794 T891 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 286794 T891 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 286794 T891 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:48440/solr
   [junit4]   2> 286794 T891 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 286795 T891 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 286795 T891 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 286797 T914 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4f6c435f name:ZooKeeperConnection Watcher:127.0.0.1:48440 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 286798 T891 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 286799 T891 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 286801 T891 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 286802 T917 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4abd3ffc name:ZooKeeperConnection Watcher:127.0.0.1:48440/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 286802 T891 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 286804 T891 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 286806 T891 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 286808 T891 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 286809 T891 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 286811 T891 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 286813 T891 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 286814 T891 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 286815 T891 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 286816 T891 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:25938_
   [junit4]   2> 286816 T891 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:25938_
   [junit4]   2> 286818 T891 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 286819 T891 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 286820 T891 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 286821 T891 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:25938_
   [junit4]   2> 286821 T891 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 286822 T891 oasc.Overseer.start Overseer (id=93545425395712003-127.0.0.1:25938_-n_0000000000) starting
   [junit4]   2> 286824 T891 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 286829 T891 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 286829 T919 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 286829 T891 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 286830 T918 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 286833 T891 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/control-001/cores
   [junit4]   2> 286835 T891 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, dataDir=data/, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/control-001/cores/collection1/, collection=control_collection, shard=, coreNodeName=, loadOnStartup=true, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/control-001/cores/collection1, transient=false, schema=schema.xml}
   [junit4]   2> 286836 T891 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/control-001/cores/collection1/
   [junit4]   2> 286836 T891 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 286837 T921 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 286837 T921 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 286838 T917 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 286838 T921 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 286839 T918 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:25938_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25938",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 0
   [junit4]   2> 286839 T918 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:25938_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25938",
   [junit4]   2> 	  "collection":"control_collection"}
   [junit4]   2> 286839 T918 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 286840 T918 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 286841 T917 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 287838 T921 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 287839 T921 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 287839 T921 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 287840 T921 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 287840 T921 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 287841 T921 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/control-001/cores/collection1/'
   [junit4]   2> 287859 T921 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 287862 T921 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 287878 T921 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 287903 T921 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 287959 T921 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 287960 T921 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 287997 T921 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 288405 T921 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 288422 T921 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 288425 T921 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 288436 T921 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 288441 T921 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 288446 T921 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 288446 T921 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 288447 T921 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 288447 T921 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 288448 T921 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 288448 T921 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 288448 T921 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 288448 T921 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 288449 T921 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/control-001/cores/collection1/data/
   [junit4]   2> 288449 T921 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4443baa7
   [junit4]   2> 288450 T921 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/control-001/cores/collection1/data
   [junit4]   2> 288450 T921 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/control-001/cores/collection1/data/index/
   [junit4]   2> 288450 T921 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 288451 T921 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/control-001/cores/collection1/data/index
   [junit4]   2> 288451 T921 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=2013953544, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 288452 T921 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6f441ae lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@e040a13),segFN=segments_1,generation=1}
   [junit4]   2> 288452 T921 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 288458 T921 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 288459 T921 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 288459 T921 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 288459 T921 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 288460 T921 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 288460 T921 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 288460 T921 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 288461 T921 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 288461 T921 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 288481 T921 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 288483 T921 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 288486 T921 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 288488 T921 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 288493 T921 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/mbeans,/update/json/docs,/admin/properties,/schema,/admin/luke,/admin/system,/config,/update/csv,/admin/threads,/update/json,standard,/admin/plugins,/get,/admin/logging,/admin/ping,/admin/segments,/update,/admin/file,/replication
   [junit4]   2> 288496 T921 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 288497 T921 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 288498 T921 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 288499 T921 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 288499 T921 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 288500 T921 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=34, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 288501 T921 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6f441ae lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@e040a13),segFN=segments_1,generation=1}
   [junit4]   2> 288501 T921 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 288501 T921 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@4ddb2a27[collection1] main
   [junit4]   2> 288502 T921 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 288502 T921 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 288503 T921 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 288503 T921 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 288503 T921 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 288503 T921 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 288504 T921 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 288504 T921 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 288505 T921 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 288505 T921 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 288506 T922 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4ddb2a27[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 288506 T921 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 288507 T925 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:25938 collection:control_collection shard:shard1
   [junit4]   2> 288508 T891 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 288508 T891 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 288508 T925 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 288511 T891 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 288512 T925 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 288512 T891 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 288513 T917 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 288513 T929 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@ec1f4de name:ZooKeeperConnection Watcher:127.0.0.1:48440/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 288514 T925 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 288514 T891 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 288514 T918 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 288514 T925 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 288514 T891 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> ASYNC  NEW_CORE C72 name=collection1 org.apache.solr.core.SolrCore@643ffd2a url=http://127.0.0.1:25938/collection1 node=127.0.0.1:25938_ C72_STATE=coll:control_collection core:collection1 props:{core=collection1, state=down, node_name=127.0.0.1:25938_, base_url=http://127.0.0.1:25938}
   [junit4]   2> 288514 T925 C:control_collection S:shard1 C72 P25938 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:25938/collection1/
   [junit4]   2> 288515 T891 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 288515 T925 C:control_collection S:shard1 C72 P25938 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 288516 T925 C:control_collection S:shard1 C72 P25938 oasc.SyncStrategy.syncToMe http://127.0.0.1:25938/collection1/ has no replicas
   [junit4]   2> 288516 T925 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:25938/collection1/ shard1
   [junit4]   2> 288516 T917 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 288516 T925 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 288518 T891 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 288519 T917 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 288520 T918 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25938",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 288623 T917 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 288623 T929 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 288672 T925 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:25938/collection1/ and leader is http://127.0.0.1:25938/collection1/
   [junit4]   2> 288673 T925 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:25938
   [junit4]   2> 288673 T925 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 288673 T925 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 288673 T925 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 288675 T917 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 288675 T918 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:25938_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25938",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 3
   [junit4]   2> 288676 T918 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:25938_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25938",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"control_collection"}
   [junit4]   2> 288779 T917 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 288779 T929 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 289496 T891 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/shard-1-001/cores/collection1
   [junit4]   2> 289498 T891 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/shard-1-001
   [junit4]   2> 289499 T891 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 289501 T891 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:50043
   [junit4]   2> 289502 T891 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/tempDir-001/jetty1, hostPort=57182, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/shard-1-001/cores, solrconfig=solrconfig.xml}
   [junit4]   2> 289502 T891 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 289502 T891 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/shard-1-001/'
   [junit4]   2> 289531 T891 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/shard-1-001/solr.xml
   [junit4]   2> 289567 T891 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/shard-1-001/cores
   [junit4]   2> 289567 T891 oasc.CoreContainer.<init> New CoreContainer 1970030467
   [junit4]   2> 289567 T891 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/shard-1-001/]
   [junit4]   2> 289568 T891 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/shard-1-001/lib
   [junit4]   2> 289568 T891 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/shard-1-001/lib).
   [junit4]   2> 289577 T891 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 289577 T891 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 289577 T891 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 289577 T891 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 289578 T891 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 289578 T891 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 289578 T891 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 289578 T891 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 289578 T891 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 289579 T891 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 289579 T891 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 289579 T891 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 289580 T891 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 289580 T891 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 289580 T891 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 289580 T891 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:48440/solr
   [junit4]   2> 289580 T891 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 289581 T891 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 289581 T891 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 289583 T941 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@76fedcde name:ZooKeeperConnection Watcher:127.0.0.1:48440 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 289584 T891 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 289584 T891 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 289586 T891 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 289869 T944 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@317f5075 name:ZooKeeperConnection Watcher:127.0.0.1:48440/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 289870 T891 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 289875 T891 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 290878 T891 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57182_
   [junit4]   2> 290879 T891 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:57182_
   [junit4]   2> 290882 T891 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 290884 T891 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/shard-1-001/cores
   [junit4]   2> 290885 T891 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, loadOnStartup=true, config=solrconfig.xml, coreNodeName=, shard=, schema=schema.xml, dataDir=data/, collection=collection1, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/shard-1-001/cores/collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/shard-1-001/cores/collection1/}
   [junit4]   2> 290886 T891 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/shard-1-001/cores/collection1/
   [junit4]   2> 290886 T891 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 290887 T945 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 290887 T945 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 290888 T945 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 290888 T917 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 290889 T918 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:57182_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57182",
   [junit4]   2> 	  "collection":"collection1"} current state version: 4
   [junit4]   2> 290890 T918 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:57182_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57182",
   [junit4]   2> 	  "collection":"collection1"}
   [junit4]   2> 290890 T918 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 290890 T918 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 290891 T929 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 290891 T917 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 290891 T944 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 291889 T945 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 291890 T945 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 291890 T945 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 291890 T945 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 291891 T945 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 291891 T945 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/shard-1-001/cores/collection1/'
   [junit4]   2> 291903 T945 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 291904 T945 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 291914 T945 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 291929 T945 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 291961 T945 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 291962 T945 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 291983 T945 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 292251 T945 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 292265 T945 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 292268 T945 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 292278 T945 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 292281 T945 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 292285 T945 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 292285 T945 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 292285 T945 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 292285 T945 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 292286 T945 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 292286 T945 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 292286 T945 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 292286 T945 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 292287 T945 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 292287 T945 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4443baa7
   [junit4]   2> 292287 T945 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/shard-1-001/cores/collection1/data
   [junit4]   2> 292288 T945 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 292288 T945 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 292288 T945 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 6DEA3E9C667F7A53-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 292288 T945 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=2013953544, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 292289 T945 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@76956412 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@49dd1690),segFN=segments_1,generation=1}
   [junit4]   2> 292289 T945 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 292294 T945 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 292294 T945 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 292295 T945 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 292295 T945 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 292295 T945 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 292295 T945 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 292295 T945 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 292296 T945 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 292296 T945 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 292316 T945 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 292318 T945 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 292320 T945 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 292321 T945 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 292327 T945 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/mbeans,/update/json/docs,/admin/properties,/schema,/admin/luke,/admin/system,/config,/update/csv,/admin/threads,/update/json,standard,/admin/plugins,/get,/admin/logging,/admin/ping,/admin/segments,/update,/admin/file,/replication
   [junit4]   2> 292329 T945 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 292330 T945 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 292331 T945 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 292332 T945 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 292332 T945 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 292333 T945 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=34, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 292333 T945 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@76956412 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@49dd1690),segFN=segments_1,generation=1}
   [junit4]   2> 292333 T945 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 292334 T945 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@177b4be2[collection1] main
   [junit4]   2> 292334 T945 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 292334 T945 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 292335 T945 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 292335 T945 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 292335 T945 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 292335 T945 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 292336 T945 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 292336 T945 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 292336 T945 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 292336 T945 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 292337 T946 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@177b4be2[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 292337 T945 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 292338 T949 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:57182 collection:collection1 shard:shard1
   [junit4]   2> 292339 T891 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 292339 T891 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 292339 T949 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 292342 T949 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 292343 T917 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 292344 T949 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 292344 T949 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C73 name=collection1 org.apache.solr.core.SolrCore@65cdc66c url=http://127.0.0.1:57182/collection1 node=127.0.0.1:57182_ C73_STATE=coll:collection1 core:collection1 props:{core=collection1, state=down, node_name=127.0.0.1:57182_, base_url=http://127.0.0.1:57182}
   [junit4]   2> 292344 T949 C:collection1 S:shard1 C73 P57182 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:57182/collection1/
   [junit4]   2> 292344 T918 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 292344 T949 C:collection1 S:shard1 C73 P57182 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 292344 T949 C:collection1 S:shard1 C73 P57182 oasc.SyncStrategy.syncToMe http://127.0.0.1:57182/collection1/ has no replicas
   [junit4]   2> 292345 T949 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:57182/collection1/ shard1
   [junit4]   2> 292345 T949 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 292347 T917 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChan

[...truncated too long message...]

  [junit4]   2> 1743366 T3562 C:control_collection S:shard1 c:collection1 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 1743367 T3562 C:control_collection S:shard1 c:collection1 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 1743367 T3562 C:control_collection S:shard1 c:collection1 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 1743368 T3562 C:control_collection S:shard1 c:collection1 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 1743370 T3562 C:control_collection S:shard1 c:collection1 oasc.CachingDirectoryFactory.close Closing StandardDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 1743370 T3562 C:control_collection S:shard1 c:collection1 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest 6DEA3E9C667F7A53-001/shard-3-001/cores/collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest 6DEA3E9C667F7A53-001/shard-3-001/cores/collection1/data;done=false>>]
   [junit4]   2> 1743370 T3562 C:control_collection S:shard1 c:collection1 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest 6DEA3E9C667F7A53-001/shard-3-001/cores/collection1/data
   [junit4]   2> 1743371 T3562 C:control_collection S:shard1 c:collection1 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest 6DEA3E9C667F7A53-001/shard-3-001/cores/collection1/data/index [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest 6DEA3E9C667F7A53-001/shard-3-001/cores/collection1/data/index;done=false>>]
   [junit4]   2> 1743371 T3562 C:control_collection S:shard1 c:collection1 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest 6DEA3E9C667F7A53-001/shard-3-001/cores/collection1/data/index
   [junit4]   2> 1743371 T3562 C:control_collection S:shard1 c:collection1 oasc.Overseer.close Overseer (id=93545513981902858-127.0.0.1:18741_ff%2Fv-n_0000000003) closing
   [junit4]   2> 1743372 T3719 oasc.Overseer$ClusterStateUpdater.run Overseer Loop exiting : 127.0.0.1:18741_ff%2Fv
   [junit4]   2> 1744873 T3717 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 1744875 T3562 C:control_collection S:shard1 c:collection1 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/ff/v,null}
   [junit4]   2> 1745078 T3562 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:18682 18682
   [junit4]   2> 1745197 T3563 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:18682 18682
   [junit4]   2> 1745198 T3563 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/aliases.json
   [junit4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent data watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		2	/solr/collections/collection1/leader_elect/shard1/election/93545513981902854-core_node1-n_0000000000
   [junit4]   2> 		2	/solr/overseer_elect/election/93545513981902854-127.0.0.1:18704_ff%2Fv-n_0000000001
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		5	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ChaosMonkeySafeLeaderTest -Dtests.method=test -Dtests.seed=6DEA3E9C667F7A53 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=zh_TW -Dtests.timezone=America/El_Salvador -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE  108s J1 | ChaosMonkeySafeLeaderTest.test <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: shard1 is not consistent.  Got 134 from http://127.0.0.1:18704/ff/v/collection1lastClient and got 118 from http://127.0.0.1:18724/ff/v/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([6DEA3E9C667F7A53:E5BE0146C88317AB]:0)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1310)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1289)
   [junit4]    > 	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:161)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1745215 T3562 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest 6DEA3E9C667F7A53-001
   [junit4]   2> 107600 T3561 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {_version_=PostingsFormat(name=LuceneVarGapDocFreqInterval), id=PostingsFormat(name=Memory doPackFST= true), a_t=Lucene50(blocksize=128), rnd_b=PostingsFormat(name=Memory doPackFST= true), a_i=PostingsFormat(name=Memory doPackFST= true)}, docValues:{}, sim=DefaultSimilarity, locale=zh_TW, timezone=America/El_Salvador
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=111989328,total=277348352
   [junit4]   2> NOTE: All tests run in this JVM: [SliceStateTest, TestClusterStateMutator, TestCryptoKeys, SimpleMLTQParserTest, DateMathParserTest, SampleTest, XmlUpdateRequestHandlerTest, ResponseLogComponentTest, TestSuggestSpellingConverter, TestReloadAndDeleteDocs, TestStandardQParsers, TestInfoStreamLogging, CloudMLTQParserTest, TestSolr4Spatial2, AliasIntegrationTest, RollingRestartTest, SuggesterFSTTest, ChangedSchemaMergeTest, TestManagedSchemaFieldResource, AddSchemaFieldsUpdateProcessorFactoryTest, TestGroupingSearch, PathHierarchyTokenizerFactoryTest, TestDynamicLoading, SuggesterWFSTTest, DefaultValueUpdateProcessorTest, TestCSVResponseWriter, SchemaVersionSpecificBehaviorTest, LeaderElectionTest, HdfsDirectoryTest, TestDFRSimilarityFactory, TestManagedStopFilterFactory, RequestHandlersTest, TestChildDocTransformer, SpatialRPTFieldTypeTest, TestSolrConfigHandler, TestDefaultSearchFieldResource, DirectUpdateHandlerOptimizeTest, PluginInfoTest, TestNoOpRegenerator, TestHashQParserPlugin, TestJsonFacets, ZkCLITest, ChaosMonkeyNothingIsSafeTest, TestRealTimeGet, DisMaxRequestHandlerTest, DistributedIntervalFacetingTest, TestLFUCache, TestPhraseSuggestions, TestRangeQuery, TestRandomDVFaceting, HighlighterConfigTest, TestCoreContainer, HdfsLockFactoryTest, JSONWriterTest, TestSolrCoreProperties, TestCollapseQParserPlugin, DateFieldTest, NoCacheHeaderTest, BadComponentTest, HttpPartitionTest, TestSolrConfigHandlerCloud, DistribDocExpirationUpdateProcessorTest, TestIntervalFaceting, ChaosMonkeySafeLeaderTest]
   [junit4] Completed on J1 in 108.63s, 1 test, 1 failure <<< FAILURES!

[...truncated 660 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 478 suites, 1900 tests, 3 errors, 1 failure, 62 ignored (22 assumptions)

Total time: 54 minutes 20 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 221041028 bytes
Compression is 0.0%
Took 38 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2837 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2837/

4 tests failed.
FAILED:  org.apache.solr.handler.component.DistributedMLTComponentTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:29409//collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:29409//collection1
	at __randomizedtesting.SeedInfo.seed([321BC4118A4F6ACA:BA4FFBCB24B30732]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:566)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:943)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:556)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:604)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:586)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:565)
	at org.apache.solr.handler.component.DistributedMLTComponentTest.test(DistributedMLTComponentTest.java:126)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 51 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:53973/mr_r/kj/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:53973/mr_r/kj/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([321BC4118A4F6ACA:BA4FFBCB24B30732]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([321BC4118A4F6ACA:BA4FFBCB24B30732]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([321BC4118A4F6ACA:BA4FFBCB24B30732]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9377 lines...]
   [junit4] Suite: org.apache.solr.handler.component.DistributedMLTComponentTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/init-core-data-001
   [junit4]   2> 168427 T727 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 168427 T727 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 169562 T727 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/control/cores/collection1
   [junit4]   2> 169565 T727 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 169568 T727 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:35911
   [junit4]   2> 169569 T727 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=35911, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/control/cores, hostContext=/}
   [junit4]   2> 169569 T727 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3972aa3f
   [junit4]   2> 169569 T727 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/control/'
   [junit4]   2> 169602 T727 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/control/solr.xml
   [junit4]   2> 169642 T727 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/control/cores
   [junit4]   2> 169643 T727 oasc.CoreContainer.<init> New CoreContainer 9403751
   [junit4]   2> 169643 T727 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/control/]
   [junit4]   2> 169643 T727 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/control/lib
   [junit4]   2> 169644 T727 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/control/lib).
   [junit4]   2> 169654 T727 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 169654 T727 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 169654 T727 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 169655 T727 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 169655 T727 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 169655 T727 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 169655 T727 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 169656 T727 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 169656 T727 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 169656 T727 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 169656 T727 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 169657 T727 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 169657 T727 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 169658 T727 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 169658 T727 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 169659 T727 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/control/cores
   [junit4]   2> 169661 T727 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/control/cores/collection1/, transient=false, config=solrconfig.xml, dataDir=data/, loadOnStartup=true, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/control/cores/collection1, configSet=collection1, name=collection1, schema=schema.xml}
   [junit4]   2> 169661 T727 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/control/cores/collection1/
   [junit4]   2> 169662 T727 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 169663 T737 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 169664 T737 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 169665 T737 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 169665 T737 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 169705 T737 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 169729 T737 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 169782 T737 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 169783 T737 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 169792 T737 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 170098 T737 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 170112 T737 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 170114 T737 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 170125 T737 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 170129 T737 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 170134 T737 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 170135 T737 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 170135 T737 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 170135 T737 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 170136 T737 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 170136 T737 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 170136 T737 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from configset /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1
   [junit4]   2> 170137 T737 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 170137 T737 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/control/cores/collection1/data/
   [junit4]   2> 170137 T737 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@37470c35
   [junit4]   2> 170139 T737 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/control/cores/collection1/data
   [junit4]   2> 170139 T737 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/control/cores/collection1/data/index/
   [junit4]   2> 170139 T737 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/control/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 170140 T737 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/control/cores/collection1/data/index
   [junit4]   2> 170140 T737 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=18, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 170141 T737 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@297e8681 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@31b0946f),segFN=segments_1,generation=1}
   [junit4]   2> 170142 T737 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 170149 T737 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 170149 T737 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 170149 T737 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 170150 T737 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 170150 T737 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 170150 T737 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 170150 T737 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 170150 T737 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 170151 T737 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 170151 T737 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 170151 T737 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 170152 T737 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 170167 T737 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 170169 T737 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 170171 T737 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 170173 T737 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 170178 T737 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 170181 T737 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /config,/terms,/update,dismax,spellCheckCompRH_Direct,/admin/plugins,/admin/luke,/debug/dump,defaults,lazy,/admin/mbeans,/schema,tvrh,/update/csv,/admin/threads,spellCheckWithWordbreak,spellCheckWithWordbreak_Direct,mock,/admin/segments,/replication,/mlt,/admin/logging,/search-facet-invariants,standard,/get,/update/json,/search-facet-def,/admin/properties,mltrh,/admin/ping,spellCheckCompRH1,spellCheckCompRH,/update/json/docs,/admin/system,/admin/file
   [junit4]   2> 170182 T737 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 170184 T737 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 170185 T737 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 170185 T737 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 170186 T737 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 170187 T737 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 170187 T737 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=524936851, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 170188 T737 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@297e8681 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@31b0946f),segFN=segments_1,generation=1}
   [junit4]   2> 170188 T737 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 170189 T737 oass.SolrIndexSearcher.<init> Opening Searcher@5832f3de[collection1] main
   [junit4]   2> 170189 T737 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 170189 T737 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 170189 T737 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 170190 T737 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 170215 T737 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 170318 T737 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 170318 T738 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 170319 T738 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 170320 T738 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 170320 T737 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 170320 T738 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 170320 T727 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 170320 T727 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 170321 T738 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 170322 T738 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 170322 T738 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 170323 T738 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 170324 T738 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 170325 T738 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5832f3de[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 172066 T727 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard0/cores/collection1
   [junit4]   2> 172070 T727 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 172073 T727 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:35912
   [junit4]   2> 172074 T727 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard0/cores, hostPort=35912, hostContext=/}
   [junit4]   2> 172074 T727 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3972aa3f
   [junit4]   2> 172075 T727 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard0/'
   [junit4]   2> 172132 T727 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard0/solr.xml
   [junit4]   2> 172207 T727 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard0/cores
   [junit4]   2> 172207 T727 oasc.CoreContainer.<init> New CoreContainer 1958509039
   [junit4]   2> 172208 T727 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard0/]
   [junit4]   2> 172208 T727 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard0/lib
   [junit4]   2> 172209 T727 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard0/lib).
   [junit4]   2> 172222 T727 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 172223 T727 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 172223 T727 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 172224 T727 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 172224 T727 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 172224 T727 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 172225 T727 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 172225 T727 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 172225 T727 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 172226 T727 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 172226 T727 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 172227 T727 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 172227 T727 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 172228 T727 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 172228 T727 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 172230 T727 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard0/cores
   [junit4]   2> 172232 T727 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard0/cores/collection1, schema=schema.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard0/cores/collection1/, config=solrconfig.xml, dataDir=data/, name=collection1, configSet=collection1, transient=false, loadOnStartup=true}
   [junit4]   2> 172232 T727 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard0/cores/collection1/
   [junit4]   2> 172233 T727 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 172234 T749 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 172235 T749 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 172236 T749 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 172236 T749 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 172287 T749 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 172317 T749 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 172381 T749 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 172382 T749 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 172390 T749 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 172663 T749 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 172676 T749 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 172678 T749 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 172688 T749 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 172692 T749 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 172695 T749 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 172696 T749 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 172696 T749 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 172696 T749 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 172696 T749 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 172697 T749 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 172697 T749 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from configset /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1
   [junit4]   2> 172697 T749 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 172698 T749 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard0/cores/collection1/data/
   [junit4]   2> 172698 T749 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@37470c35
   [junit4]   2> 172699 T749 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard0/cores/collection1/data
   [junit4]   2> 172699 T749 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard0/cores/collection1/data/index/
   [junit4]   2> 172699 T749 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard0/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 172699 T749 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard0/cores/collection1/data/index
   [junit4]   2> 172700 T749 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=18, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 172700 T749 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@641573b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2bc5e5b2),segFN=segments_1,generation=1}
   [junit4]   2> 172701 T749 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 172709 T749 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 172709 T749 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 172709 T749 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 172710 T749 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 172710 T749 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 172710 T749 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 172710 T749 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 172710 T749 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 172711 T749 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 172711 T749 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 172711 T749 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 172711 T749 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 172724 T749 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 172727 T749 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 172728 T749 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 172730 T749 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 172737 T749 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 172741 T749 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /config,/terms,/update,dismax,spellCheckCompRH_Direct,/admin/plugins,/admin/luke,/debug/dump,defaults,lazy,/admin/mbeans,/schema,tvrh,/update/csv,/admin/threads,spellCheckWithWordbreak,spellCheckWithWordbreak_Direct,mock,/admin/segments,/replication,/mlt,/admin/logging,/search-facet-invariants,standard,/get,/update/json,/search-facet-def,/admin/properties,mltrh,/admin/ping,spellCheckCompRH1,spellCheckCompRH,/update/json/docs,/admin/system,/admin/file
   [junit4]   2> 172742 T749 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 172743 T749 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 172744 T749 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 172745 T749 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 172746 T749 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 172746 T749 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 172747 T749 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=524936851, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 172748 T749 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@641573b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2bc5e5b2),segFN=segments_1,generation=1}
   [junit4]   2> 172748 T749 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 172748 T749 oass.SolrIndexSearcher.<init> Opening Searcher@6b190e5b[collection1] main
   [junit4]   2> 172749 T749 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 172749 T749 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 172749 T749 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 172749 T749 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 172773 T749 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 172827 T749 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 172828 T750 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 172829 T750 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 172829 T750 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 172829 T750 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 172830 T749 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 172830 T750 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 172831 T727 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 172831 T727 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 172832 T750 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 172833 T750 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 172834 T750 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 172835 T750 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 172837 T750 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6b190e5b[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 173947 T727 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard1/cores/collection1
   [junit4]   2> 173949 T727 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 173952 T727 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:10531
   [junit4]   2> 173952 T727 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard1/cores, hostContext=/, hostPort=10531}
   [junit4]   2> 173952 T727 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3972aa3f
   [junit4]   2> 173953 T727 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard1/'
   [junit4]   2> 173987 T727 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard1/solr.xml
   [junit4]   2> 174030 T727 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard1/cores
   [junit4]   2> 174030 T727 oasc.CoreContainer.<init> New CoreContainer 666341314
   [junit4]   2> 174031 T727 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard1/]
   [junit4]   2> 174031 T727 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard1/lib
   [junit4]   2> 174032 T727 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard1/lib).
   [junit4]   2> 174041 T727 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 174042 T727 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 174042 T727 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 174042 T727 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 174042 T727 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 174043 T727 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 174043 T727 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 174043 T727 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 174043 T727 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 174044 T727 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 174044 T727 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 174044 T727 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 174045 T727 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 174045 T727 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 174045 T727 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 174046 T727 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard1/cores
   [junit4]   2> 174048 T727 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard1/cores/collection1/, transient=false, config=solrconfig.xml, configSet=collection1, dataDir=data/, name=collection1, schema=schema.xml, loadOnStartup=true, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard1/cores/collection1}
   [junit4]   2> 174048 T727 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard1/cores/collection1/
   [junit4]   2> 174049 T727 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 174050 T761 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 174051 T761 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 174052 T761 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 174052 T761 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 174096 T761 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 174124 T761 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 174186 T761 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 174187 T761 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 174197 T761 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 174559 T761 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 174576 T761 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 174579 T761 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 174593 T761 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 174598 T761 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 174602 T761 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 174603 T761 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 174603 T761 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 174603 T761 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 174604 T761 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 174604 T761 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 174605 T761 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from configset /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1
   [junit4]   2> 174605 T761 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 174605 T761 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard1/cores/collection1/data/
   [junit4]   2> 174606 T761 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@37470c35
   [junit4]   2> 174606 T761 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard1/cores/collection1/data
   [junit4]   2> 174607 T761 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard1/cores/collection1/data/index/
   [junit4]   2> 174607 T761 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard1/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 174607 T761 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard1/cores/collection1/data/index
   [junit4]   2> 174608 T761 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=18, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 174609 T761 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@798c3e9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3e65f9e4),segFN=segments_1,generation=1}
   [junit4]   2> 174609 T761 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 174617 T761 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 174618 T761 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 174618 T761 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 174618 T761 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 174619 T761 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 174619 T761 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 174619 T761 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 174620 T761 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 174620 T761 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 174620 T761 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 174621 T761 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 174621 T761 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 174640 T761 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 174643 T761 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 174646 T761 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 174648 T761 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 174657 T761 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 174662 T761 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /config,/terms,/update,dismax,spellCheckCompRH_Direct,/admin/plugins,/admin/luke,/debug/dump,defaults,lazy,/admin/mbeans,/schema,tvrh,/update/csv,/admin/threads,spellCheckWithWordbreak,spellCheckWithWordbreak_Direct,mock,/admin/segments,/replication,/mlt,/admin/logging,/search-facet-invariants,standard,/get,/update/json,/search-facet-def,/admin/properties,mltrh,/admin/ping,spellCheckCompRH1,spellCheckCompRH,/update/json/docs,/admin/system,/admin/file
   [junit4]   2> 174664 T761 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 174665 T761 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 174667 T761 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 174667 T761 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 174669 T761 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 174669 T761 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 174670 T761 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=524936851, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 174671 T761 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@798c3e9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3e65f9e4),segFN=segments_1,generation=1}
   [junit4]   2> 174671 T761 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 174672 T761 oass.SolrIndexSearcher.<init> Opening Searcher@696cc212[collection1] main
   [junit4]   2> 174672 T761 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 174673 T761 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 174673 T761 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 174673 T761 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 174709 T761 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 174769 T761 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 174769 T762 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 174770 T762 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 174771 T762 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 174771 T762 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 174771 T761 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 174772 T762 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 174772 T727 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 174772 T727 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 174773 T762 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 174774 T762 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 174775 T762 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 174776 T762 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 174778 T762 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@696cc212[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 176062 T727 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard2/cores/collection1
   [junit4]   2> 176064 T727 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 176066 T727 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:29409
   [junit4]   2> 176066 T727 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=29409, hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard2/cores}
   [junit4]   2> 176067 T727 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3972aa3f
   [junit4]   2> 176067 T727 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard2/'
   [junit4]   2> 176101 T727 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard2/solr.xml
   [junit4]   2> 176138 T727 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard2/cores
   [junit4]   2> 176138 T727 oasc.CoreContainer.<init> New CoreContainer 1628616442
   [junit4]   2> 176138 T727 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard2/]
   [junit4]   2> 176139 T727 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard2/lib
   [junit4]   2> 176139 T727 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard2/lib).
   [junit4]   2> 176149 T727 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 176149 T727 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 176149 T727 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 176150 T727 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 176150 T727 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 176150 T727 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 176150 T727 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 176151 T727 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 176151 T727 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 176151 T727 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 176151 T727 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 176152 T727 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 176152 T727 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 176152 T727 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 176153 T727 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 176154 T727 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard2/cores
   [junit4]   2> 176155 T727 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {loadOnStartup=true, transient=false, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard2/cores/collection1/, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedMLTComponentTest 321BC4118A4F6ACA-001/tempDir-001/shard2/cores/collection1, configSet=collection1, name=collection1, dataDir=data/, schema=schema.xml}
   [junit4]   2> 176156 T727 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.handler.component.Di

[...truncated too long message...]

195 54195
   [junit4]   2> 1650303 T4023 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:54195 54195
   [junit4]   2> 1650304 T4023 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 		2	/solr/collections/collection1/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1650305 T4022 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:54209/ct_cc/sk
   [junit4]   2> 1650305 T4022 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:54200/ct_cc/sk
   [junit4]   2> 1650305 T4022 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:54224/ct_cc/sk
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=321BC4118A4F6ACA -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ar_DZ -Dtests.timezone=Asia/Kuching -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   67.3s J0 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([321BC4118A4F6ACA:BA4FFBCB24B30732]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 1650318 T4022 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 321BC4118A4F6ACA-001
   [junit4]   2> 67319 T4021 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {_version_=PostingsFormat(name=LuceneFixedGap), text=PostingsFormat(name=Memory doPackFST= false), multiDefault=Lucene50(blocksize=128), timestamp=PostingsFormat(name=LuceneFixedGap), id=Lucene50(blocksize=128), intDefault=PostingsFormat(name=LuceneFixedGap), range_facet_l=PostingsFormat(name=LuceneFixedGap), a_t=PostingsFormat(name=LuceneFixedGap)}, docValues:{timestamp=DocValuesFormat(name=Memory)}, sim=DefaultSimilarity, locale=ar_DZ, timezone=Asia/Kuching
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=106033264,total=320864256
   [junit4]   2> NOTE: All tests run in this JVM: [ConvertedLegacyTest, TestQueryUtils, TestEmbeddedSolrServer, OverseerRolesTest, TestSolrXml, TestInitParams, SuggesterWFSTTest, TestImplicitCoreProperties, TestMiniSolrCloudCluster, AnalysisErrorHandlingTest, CoreAdminCreateDiscoverTest, TestConfigReload, DistributedMLTComponentTest, TestSolr4Spatial, TestSchemaResource, TestBlobHandler, TestFuzzyAnalyzedSuggestions, TestCoreDiscovery, TestRequestStatusCollectionAPI, SharedFSAutoReplicaFailoverTest, TestBadConfig, TestSearcherReuse, TestNonDefinedSimilarityFactory, TestIndexingPerformance, TestExceedMaxTermLength, FileBasedSpellCheckerTest, BJQParserTest, TestObjectReleaseTracker, DistributedQueryComponentOptimizationTest, DeleteInactiveReplicaTest, TestTrackingShardHandlerFactory, TestUniqueKeyFieldResource, PreAnalyzedFieldTest, TestDocBasedVersionConstraints, VMParamsZkACLAndCredentialsProvidersTest, XmlUpdateRequestHandlerTest, TestFastWriter, SoftAutoCommitTest, SuggesterTest, OverseerCollectionProcessorTest, TestReqParamsAPI, IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestQuerySenderListener, StatelessScriptUpdateProcessorFactoryTest, SuggesterTSTTest, TestDocumentBuilder, TestManagedSchema, TestRandomDVFaceting, RequestLoggingTest, ShardRoutingTest, TestLFUCache, TestReRankQParserPlugin, StandardRequestHandlerTest, ExitableDirectoryReaderTest, TestMergePolicyConfig, HttpPartitionTest, ShardSplitTest, CustomCollectionTest, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J0 in 67.86s, 1 test, 1 error <<< FAILURES!

[...truncated 721 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 477 suites, 1898 tests, 4 errors, 62 ignored (22 assumptions)

Total time: 53 minutes 55 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 206755073 bytes
Compression is 0.0%
Took 39 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2836 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2836/

4 tests failed.
REGRESSION:  org.apache.solr.handler.component.DistributedMLTComponentTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:45361/hxqr/o/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:45361/hxqr/o/collection1
	at __randomizedtesting.SeedInfo.seed([77E3E74BCE072F65:FFB7D89160FB429D]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:594)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:943)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:556)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:604)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:586)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:565)
	at org.apache.solr.handler.component.DistributedMLTComponentTest.test(DistributedMLTComponentTest.java:126)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 51 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:53075/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:53075/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([77E3E74BCE072F65:FFB7D89160FB429D]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([77E3E74BCE072F65:FFB7D89160FB429D]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([77E3E74BCE072F65:FFB7D89160FB429D]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9849 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/init-core-data-001
   [junit4]   2> 1046359 T2275 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 1046367 T2275 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1046368 T2276 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1046468 T2275 oasc.ZkTestServer.run start zk server on port:18696
   [junit4]   2> 1046469 T2275 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1046470 T2275 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1046475 T2283 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@49a4fe48 name:ZooKeeperConnection Watcher:127.0.0.1:18696 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1046475 T2275 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1046475 T2275 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1046475 T2275 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1046478 T2275 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1046480 T2275 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1046481 T2286 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@73163a30 name:ZooKeeperConnection Watcher:127.0.0.1:18696/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1046481 T2275 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1046482 T2275 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1046482 T2275 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1046485 T2275 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1046486 T2275 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1046487 T2275 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1046489 T2275 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1046489 T2275 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1046492 T2275 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1046493 T2275 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1046595 T2275 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1046595 T2275 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1046597 T2275 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1046597 T2275 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1046598 T2275 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1046599 T2275 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1046600 T2275 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1046601 T2275 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1046602 T2275 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1046602 T2275 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1046604 T2275 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1046604 T2275 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1046606 T2275 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1046606 T2275 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1046607 T2275 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1046608 T2275 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1046609 T2275 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1046610 T2275 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1047581 T2275 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/control-001/cores/collection1
   [junit4]   2> 1047583 T2275 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1047587 T2275 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:18700
   [junit4]   2> 1047587 T2275 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=18699, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/control-001/cores, hostContext=/}
   [junit4]   2> 1047587 T2275 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3972aa3f
   [junit4]   2> 1047588 T2275 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/control-001/'
   [junit4]   2> 1047617 T2275 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/control-001/solr.xml
   [junit4]   2> 1047654 T2275 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/control-001/cores
   [junit4]   2> 1047654 T2275 oasc.CoreContainer.<init> New CoreContainer 515935613
   [junit4]   2> 1047655 T2275 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/control-001/]
   [junit4]   2> 1047655 T2275 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/control-001/lib
   [junit4]   2> 1047655 T2275 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/control-001/lib).
   [junit4]   2> 1047677 T2275 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1047677 T2275 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1047677 T2275 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1047678 T2275 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1047678 T2275 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1047678 T2275 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1047679 T2275 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1047679 T2275 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1047679 T2275 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1047679 T2275 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1047680 T2275 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1047680 T2275 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1047681 T2275 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1047681 T2275 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1047681 T2275 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1047682 T2275 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:18696/solr
   [junit4]   2> 1047682 T2275 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1047682 T2275 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1047683 T2275 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1047686 T2298 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7c74eab2 name:ZooKeeperConnection Watcher:127.0.0.1:18696 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1047687 T2275 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1047687 T2275 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1047690 T2275 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1047691 T2301 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@74576c13 name:ZooKeeperConnection Watcher:127.0.0.1:18696/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1047692 T2275 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1047693 T2275 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1047695 T2275 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1047697 T2275 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1047699 T2275 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1047702 T2275 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1047704 T2275 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1047705 T2275 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1047706 T2275 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1047707 T2275 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:18699_
   [junit4]   2> 1047708 T2275 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:18699_
   [junit4]   2> 1047709 T2275 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1047710 T2275 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1047712 T2275 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1047713 T2275 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:18699_
   [junit4]   2> 1047713 T2275 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1047715 T2275 oasc.Overseer.start Overseer (id=93543379734036483-127.0.0.1:18699_-n_0000000000) starting
   [junit4]   2> 1047716 T2275 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1047721 T2275 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 1047722 T2303 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1047722 T2275 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1047722 T2302 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1047726 T2275 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/control-001/cores
   [junit4]   2> 1047728 T2275 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {dataDir=data/, schema=schema.xml, transient=false, coreNodeName=, collection=control_collection, loadOnStartup=true, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/control-001/cores/collection1/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/control-001/cores/collection1, shard=, name=collection1, config=solrconfig.xml}
   [junit4]   2> 1047728 T2275 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/control-001/cores/collection1/
   [junit4]   2> 1047729 T2275 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1047730 T2305 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1047730 T2305 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1047731 T2305 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1047731 T2301 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1047732 T2302 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:18699_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:18699",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1"} current state version: 0
   [junit4]   2> 1047732 T2302 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:18699_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:18699",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 1047733 T2302 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 1047733 T2302 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1047734 T2301 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1048731 T2305 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1048732 T2305 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1048732 T2305 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1048733 T2305 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1048733 T2305 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1048733 T2305 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/control-001/cores/collection1/'
   [junit4]   2> 1048751 T2305 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1048754 T2305 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1048764 T2305 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1048786 T2305 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 1048834 T2305 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1048936 T2305 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1048947 T2305 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1049369 T2305 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1049386 T2305 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1049390 T2305 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1049399 T2305 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1049404 T2305 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1049408 T2305 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1049409 T2305 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1049409 T2305 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1049409 T2305 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1049410 T2305 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1049410 T2305 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1049410 T2305 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1049411 T2305 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1049411 T2305 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/control-001/cores/collection1/data/
   [junit4]   2> 1049411 T2305 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@62701466
   [junit4]   2> 1049412 T2305 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/control-001/cores/collection1/data
   [junit4]   2> 1049412 T2305 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/control-001/cores/collection1/data/index/
   [junit4]   2> 1049413 T2305 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1049413 T2305 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/control-001/cores/collection1/data/index
   [junit4]   2> 1049413 T2305 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=6.8369140625, floorSegmentMB=1.3408203125, forceMergeDeletesPctAllowed=28.719063893793074, segmentsPerTier=47.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1897029406179554
   [junit4]   2> 1049414 T2305 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=RawDirectoryWrapper(RAMDirectory@34e36f5c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3fa41c24),segFN=segments_1,generation=1}
   [junit4]   2> 1049415 T2305 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1049420 T2305 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1049420 T2305 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1049420 T2305 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1049421 T2305 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1049421 T2305 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1049421 T2305 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1049422 T2305 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1049422 T2305 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1049422 T2305 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1049449 T2305 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1049452 T2305 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1049454 T2305 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1049457 T2305 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1049462 T2305 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: standard,/admin/file,/update/json/docs,/replication,/schema,/admin/segments,/admin/luke,/update/json,/admin/threads,/admin/ping,/admin/plugins,/admin/system,/update,/update/csv,/admin/properties,/admin/logging,/get,/config,/admin/mbeans
   [junit4]   2> 1049464 T2305 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1049466 T2305 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1049466 T2305 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1049468 T2305 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1049469 T2305 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1049469 T2305 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=2092292617, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1049470 T2305 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=RawDirectoryWrapper(RAMDirectory@34e36f5c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3fa41c24),segFN=segments_1,generation=1}
   [junit4]   2> 1049470 T2305 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1049471 T2305 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@7f197907[collection1] main
   [junit4]   2> 1049471 T2305 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1049472 T2305 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1049472 T2305 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1049472 T2305 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1049473 T2305 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1049473 T2305 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1049473 T2305 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1049474 T2305 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1049474 T2305 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1049474 T2305 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1049475 T2306 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7f197907[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1049475 T2305 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1049476 T2309 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:18699 collection:control_collection shard:shard1
   [junit4]   2> 1049476 T2275 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 1049477 T2275 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1049477 T2309 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1049478 T2275 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1049478 T2275 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1049480 T2313 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@631a1a3c name:ZooKeeperConnection Watcher:127.0.0.1:18696/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1049480 T2275 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1049481 T2275 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1049481 T2275 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1049483 T2309 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1049494 T2301 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1049494 T2309 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1049494 T2275 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1049496 T2302 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 1049496 T2309 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 1049496 T2275 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 1049498 T2275 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> ASYNC  NEW_CORE C325 name=collection1 org.apache.solr.core.SolrCore@163d3890 url=http://127.0.0.1:18699/collection1 node=127.0.0.1:18699_ C325_STATE=coll:control_collection core:collection1 props:{core=collection1, state=down, node_name=127.0.0.1:18699_, base_url=http://127.0.0.1:18699}
   [junit4]   2> 1049497 T2309 C:control_collection S:shard1 C325 P18699 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:18699/collection1/
   [junit4]   2> 1049500 T2309 C:control_collection S:shard1 C325 P18699 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1049500 T2275 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1049500 T2309 C:control_collection S:shard1 C325 P18699 oasc.SyncStrategy.syncToMe http://127.0.0.1:18699/collection1/ has no replicas
   [junit4]   2> 1049500 T2309 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:18699/collection1/ shard1
   [junit4]   2> 1049501 T2301 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1049501 T2313 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1049501 T2309 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1049502 T2316 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@51965c17 name:ZooKeeperConnection Watcher:127.0.0.1:18696/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1049502 T2275 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1049503 T2275 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1049505 T2301 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1049506 T2302 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 1049506 T2302 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 1049507 T2302 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:18699",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 1049508 T2302 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 1049508 T2301 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1049508 T2313 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1049510 T2301 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1049511 T2313 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1049511 T2301 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1049556 T2309 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:18699/collection1/ and leader is http://127.0.0.1:18699/collection1/
   [junit4]   2> 1049557 T2309 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:18699
   [junit4]   2> 1049557 T2309 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1049557 T2309 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1049557 T2309 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1049558 T2301 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1049559 T2302 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:18699_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:18699",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"3"} current state version: 4
   [junit4]   2> 1049559 T2302 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:18699_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:18699",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"3"}
   [junit4]   2> 1049662 T2301 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1049662 T2313 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1050826 T2275 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/shard-1-001/cores/collection1
   [junit4]   2> 1050828 T2275 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/shard-1-001
   [junit4]   2> 1050829 T2275 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1050832 T2275 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:18707
   [junit4]   2> 1050832 T2275 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=18706, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/shard-1-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/}
   [junit4]   2> 1050832 T2275 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3972aa3f
   [junit4]   2> 1050833 T2275 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/shard-1-001/'
   [junit4]   2> 1050865 T2275 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/shard-1-001/solr.xml
   [junit4]   2> 1050908 T2275 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/shard-1-001/cores
   [junit4]   2> 1050909 T2275 oasc.CoreContainer.<init> New CoreContainer 61342714
   [junit4]   2> 1050909 T2275 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/shard-1-001/]
   [junit4]   2> 1050909 T2275 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/shard-1-001/lib
   [junit4]   2> 1050909 T2275 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/shard-1-001/lib).
   [junit4]   2> 1050918 T2275 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1050919 T2275 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1050919 T2275 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1050919 T2275 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1050919 T2275 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1050919 T2275 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1050920 T2275 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1050920 T2275 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1050920 T2275 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1050920 T2275 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1050920 T2275 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1050921 T2275 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1050921 T2275 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1050921 T2275 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1050922 T2275 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1050922 T2275 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:18696/solr
   [junit4]   2> 1050922 T2275 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1050922 T2275 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1050923 T2275 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1050924 T2328 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2a2eecb7 name:ZooKeeperConnection Watcher:127.0.0.1:18696 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1050925 T2275 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1050926 T2275 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1050927 T2275 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1050929 T2331 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2213b480 name:ZooKeeperConnection Watcher:127.0.0.1:18696/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1050929 T2275 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1050933 T2275 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1051936 T2275 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:18706_
   [junit4]   2> 1051937 T2275 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:18706_
   [junit4]   2> 1051940 T2275 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1051942 T2275 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/shard-1-001/cores
   [junit4]   2> 1051944 T2275 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/shard-1-001/cores/collection1/, shard=, config=solrconfig.xml, collection=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/shard-1-001/cores/collection1, dataDir=data/, schema=schema.xml, name=collection1, coreNodeName=, transient=false, loadOnStartup=true}
   [junit4]   2> 1051944 T2275 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/shard-1-001/cores/collection1/
   [junit4]   2> 1051945 T2275 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1051947 T2332 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1051947 T2332 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1051948 T2301 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1051948 T2332 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 1051949 T2332 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 1051949 T2302 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:18706_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:18706",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"3"} current state version: 5
   [junit4]   2> 1051949 T2332 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 1051949 T2302 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:18706_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:18706",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "numShards":"3"}
   [junit4]   2> 1051950 T2302 oasco.ReplicaMutator.updateState Collection already exists with numShards=3
   [junit4]   2> 1051950 T2302 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard3
   [junit4]   2> 1051950 T2332 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 1051950 T2332 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1051951 T2302 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 1051952 T2331 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1051952 T2331 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 1052951 T2332 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1052952 T2332 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1052952 T2332 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1052952 T2332 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1052953 T2332 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1052953 T2332 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/shard-1-001/cores/collection1/'
   [junit4]   2> 1052965 T2332 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1052967 T2332 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1052974 T2332 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1052989 T2332 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 1053021 T2332 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1053122 T2332 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1053128 T2332 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1053420 T2332 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1053432 T2332 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1053435 T2332 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1053441 T2332 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1053444 T2332 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1053447 T2332 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1053447 T2332 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1053448 T2332 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1053448 T2332 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1053448 T2332 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1053448 T2332 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1053448 T2332 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1053449 T2332 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1053449 T2332 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 1053449 T2332 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@62701466
   [junit4]   2> 1053450 T2332 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/shard-1-001/cores/collection1/data
   [junit4]   2> 1053450 T2332 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 1053450 T2332 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1053450 T2332 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 77E3E74BCE072F65-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 1053451 T2332 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=6.8369140625, floorSegmentMB=1.3408203125, forceMergeDeletesPctAllowed=28.719063893793074, segmentsPerTier=47.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1897029406179554
   [junit4]   2> 1053451 T2332 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=RawDirectoryWrapper(RAMDirectory@6a40d7a4 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@470d867b),segFN=segments_1,generation=1}
   [junit4]   2> 1053452 T2332 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1053456 T2332 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1053456 T2332 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1053456 T2332 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1053457 T2332 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1053457 T2332 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1053457 T2332 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1053457 T2332 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1053457 T2332 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1053458 T2332 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1053477 T2332 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1053479 T2332 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1053480 T2332 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1053482 T2332 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1053489 T2332 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: standard,/admin/file,/update/json/docs,/replication,/schema,/admin/segments,/admin/luke,/update/json,/admin/threads,/admin/ping,/admin/plugins,/admin/system,/update,/update/csv,/admin/properties,/admin/logging,/get,/config,/admin/mbeans
   [junit4]   2> 1053491 T2332 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1053493 T2332 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1053493 T2332 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1053495 T2332 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1053495 T2332 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1053498 T2332 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.ap

[...truncated too long message...]

r.component.DistributedMLTComponentTest 77E3E74BCE072F65-001/tempDir-001/shard2/cores/collection1/data/index
   [junit4]   2> 2003421 T5199 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.handler.component.DistributedMLTComponentTest 77E3E74BCE072F65-001/tempDir-001/shard2/cores/collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.handler.component.DistributedMLTComponentTest 77E3E74BCE072F65-001/tempDir-001/shard2/cores/collection1/data;done=false>>]
   [junit4]   2> 2003422 T5199 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.handler.component.DistributedMLTComponentTest 77E3E74BCE072F65-001/tempDir-001/shard2/cores/collection1/data
   [junit4]   2> 2003423 T5199 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/hxqr/o,null}
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=DistributedMLTComponentTest -Dtests.method=test -Dtests.seed=77E3E74BCE072F65 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=fr_CH -Dtests.timezone=Asia/Sakhalin -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR    102s J2 | DistributedMLTComponentTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:45361/hxqr/o/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([77E3E74BCE072F65:FFB7D89160FB429D]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:594)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:943)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:556)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:604)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:586)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:565)
   [junit4]    > 	at org.apache.solr.handler.component.DistributedMLTComponentTest.test(DistributedMLTComponentTest.java:126)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: java.net.SocketTimeoutException: Read timed out
   [junit4]    > 	at java.net.SocketInputStream.socketRead0(Native Method)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:152)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:122)
   [junit4]    > 	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
   [junit4]    > 	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
   [junit4]    > 	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
   [junit4]    > 	... 51 more
   [junit4]   2> 2003631 T5199 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.handler.component.DistributedMLTComponentTest 77E3E74BCE072F65-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {intDefault=FST50, timestamp=FSTOrd50, multiDefault=FSTOrd50, lowerfilt1=PostingsFormat(name=MockRandom), lowerfilt=FSTOrd50, _version_=FST50, range_facet_l=FSTOrd50, lowerfilt1and2=FSTOrd50, id=PostingsFormat(name=LuceneVarGapFixedInterval)}, docValues:{timestamp=DocValuesFormat(name=Asserting)}, sim=DefaultSimilarity, locale=fr_CH, timezone=Asia/Sakhalin
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=156987936,total=353894400
   [junit4]   2> NOTE: All tests run in this JVM: [TestRTGBase, SearchHandlerTest, TestRandomDVFaceting, DistributedQueryElevationComponentTest, SpatialHeatmapFacetsTest, URLClassifyProcessorTest, OverseerRolesTest, TestSolrDeletionPolicy1, IndexSchemaRuntimeFieldTest, XmlUpdateRequestHandlerTest, DistributedQueryComponentOptimizationTest, TestDefaultSearchFieldResource, TestValueSourceCache, TestSolrQueryParserResource, SolrCoreTest, DeleteInactiveReplicaTest, ExternalFileFieldSortTest, TestUpdate, NoCacheHeaderTest, HdfsLockFactoryTest, HighlighterTest, TestCollationFieldDocValues, TestExactStatsCache, FileBasedSpellCheckerTest, TestReqParamsAPI, TestFreeTextSuggestions, MoreLikeThisHandlerTest, DistributedFacetPivotSmallAdvancedTest, TestJettySolrRunner, TestTrie, SignatureUpdateProcessorFactoryTest, TestCollapseQParserPlugin, TestDistributedSearch, PluginInfoTest, TestCodecSupport, TestGroupingSearch, DistributedQueryComponentCustomSortTest, TestFuzzyAnalyzedSuggestions, TestManagedResource, TestSchemaVersionResource, TestSolr4Spatial2, PrimUtilsTest, DebugComponentTest, TestFieldSortValues, TestFileDictionaryLookup, StandardRequestHandlerTest, SyncSliceTest, TestSweetSpotSimilarityFactory, AssignTest, SuggesterFSTTest, TestRandomFaceting, CoreAdminRequestStatusTest, TestPerFieldSimilarity, TestClusterStateMutator, TestStressRecovery, TestBulkSchemaAPI, TestCloudSchemaless, RecoveryZkTest, ShardSplitTest, FullSolrCloudDistribCmdsTest, DistribDocExpirationUpdateProcessorTest, MigrateRouteKeyTest, ExternalCollectionsTest, RollingRestartTest, OverseerStatusTest, TestDistribDocBasedVersion, LeaderElectionTest, TestRecovery, TestTrackingShardHandlerFactory, DistributedFacetPivotSmallTest, SolrCmdDistributorTest, DistributedMLTComponentTest]
   [junit4] Completed on J2 in 102.31s, 1 test, 1 error <<< FAILURES!

[...truncated 366 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 476 suites, 1896 tests, 4 errors, 62 ignored (22 assumptions)

Total time: 53 minutes 14 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 213719443 bytes
Compression is 0.0%
Took 40 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2835 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2835/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:37053/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37053/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([91A5A3C94AE73B0D:19F19C13E41B56F5]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([91A5A3C94AE73B0D:19F19C13E41B56F5]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([91A5A3C94AE73B0D:19F19C13E41B56F5]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9411 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/init-core-data-001
   [junit4]   2> 194270 T495 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /_/y
   [junit4]   2> 194275 T495 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 194276 T496 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 194376 T495 oasc.ZkTestServer.run start zk server on port:21028
   [junit4]   2> 194377 T495 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 194378 T495 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 194382 T503 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@26f8194b name:ZooKeeperConnection Watcher:127.0.0.1:21028 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 194382 T495 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 194383 T495 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 194383 T495 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 194386 T495 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 194387 T495 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 194388 T506 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@583a59d1 name:ZooKeeperConnection Watcher:127.0.0.1:21028/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 194389 T495 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 194389 T495 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 194390 T495 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 194393 T495 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 194396 T495 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 194398 T495 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 194402 T495 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 194402 T495 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 194407 T495 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 194407 T495 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 194510 T495 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 194511 T495 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 194514 T495 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 194514 T495 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 194517 T495 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 194518 T495 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 194521 T495 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 194521 T495 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 194524 T495 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 194525 T495 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 194528 T495 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 194528 T495 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 194532 T495 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 194532 T495 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 194535 T495 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 194536 T495 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 194539 T495 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 194539 T495 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 196241 T495 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/control-001/cores/collection1
   [junit4]   2> 196247 T495 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 196250 T495 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:21034
   [junit4]   2> 196251 T495 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/tempDir-001/control/data, hostContext=/_/y, hostPort=21033, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/control-001/cores}
   [junit4]   2> 196251 T495 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 196252 T495 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/control-001/'
   [junit4]   2> 196315 T495 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/control-001/solr.xml
   [junit4]   2> 196376 T495 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/control-001/cores
   [junit4]   2> 196376 T495 oasc.CoreContainer.<init> New CoreContainer 1104313762
   [junit4]   2> 196377 T495 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/control-001/]
   [junit4]   2> 196377 T495 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/control-001/lib
   [junit4]   2> 196378 T495 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/control-001/lib).
   [junit4]   2> 196391 T495 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 196391 T495 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 196392 T495 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 196392 T495 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 196392 T495 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 196393 T495 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 196393 T495 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 196393 T495 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 196394 T495 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 196394 T495 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 196394 T495 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 196395 T495 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 196396 T495 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 196396 T495 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 196396 T495 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 196397 T495 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:21028/solr
   [junit4]   2> 196397 T495 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 196397 T495 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 196398 T495 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 196401 T518 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@60cd0e77 name:ZooKeeperConnection Watcher:127.0.0.1:21028 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 196401 T495 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 196402 T495 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 196421 T495 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 196423 T521 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1936e788 name:ZooKeeperConnection Watcher:127.0.0.1:21028/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 196423 T495 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 196438 T495 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 196447 T495 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 196457 T495 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 196461 T495 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 196465 T495 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 196475 T495 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 196478 T495 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 196481 T495 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 196483 T495 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:21033__%2Fy
   [junit4]   2> 196491 T495 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:21033__%2Fy
   [junit4]   2> 196495 T495 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 196497 T495 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 196500 T495 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 196509 T495 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:21033__%2Fy
   [junit4]   2> 196509 T495 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 196512 T495 oasc.Overseer.start Overseer (id=93542382333788163-127.0.0.1:21033__%2Fy-n_0000000000) starting
   [junit4]   2> 196533 T495 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 196570 T495 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 196593 T523 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 196593 T495 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 196600 T522 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 196600 T495 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/control-001/cores
   [junit4]   2> 196617 T495 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {shard=, loadOnStartup=true, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/control-001/cores/collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/control-001/cores/collection1/, config=solrconfig.xml, transient=false, coreNodeName=, collection=control_collection, name=collection1, dataDir=data/, schema=schema.xml}
   [junit4]   2> 196618 T495 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/control-001/cores/collection1/
   [junit4]   2> 196618 T495 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 196620 T525 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 196620 T525 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 196626 T521 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 196626 T525 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 196633 T522 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:21033/_/y",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:21033__%2Fy",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state"} current state version: 0
   [junit4]   2> 196676 T522 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:21033/_/y",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:21033__%2Fy",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 196683 T522 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 196700 T522 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 196703 T521 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 197627 T525 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 197627 T525 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 197628 T525 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 197628 T525 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 197630 T525 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 197630 T525 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/control-001/cores/collection1/'
   [junit4]   2> 197660 T525 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 197675 T525 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 197695 T525 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 197713 T525 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 197758 T525 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 197860 T525 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 197872 T525 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 198284 T525 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 198302 T525 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 198305 T525 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 198324 T525 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 198329 T525 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 198334 T525 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 198336 T525 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 198336 T525 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 198337 T525 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 198338 T525 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 198338 T525 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 198339 T525 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 198339 T525 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 198339 T525 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/control-001/cores/collection1/data/
   [junit4]   2> 198340 T525 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@588ce5d4
   [junit4]   2> 198341 T525 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/control-001/cores/collection1/data
   [junit4]   2> 198341 T525 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/control-001/cores/collection1/data/index/
   [junit4]   2> 198341 T525 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 198342 T525 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/control-001/cores/collection1/data/index
   [junit4]   2> 198342 T525 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=40, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.15185581651979377]
   [junit4]   2> 198343 T525 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7ed2caf8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@410ba0d4),segFN=segments_1,generation=1}
   [junit4]   2> 198343 T525 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 198351 T525 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 198352 T525 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 198352 T525 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 198353 T525 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 198353 T525 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 198353 T525 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 198354 T525 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 198354 T525 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 198354 T525 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 198373 T525 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 198376 T525 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 198378 T525 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 198380 T525 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 198387 T525 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /replication,/update/csv,/admin/threads,/admin/mbeans,/admin/plugins,/admin/luke,/update/json,/admin/file,/admin/logging,/config,/admin/system,/get,/admin/segments,/admin/ping,/schema,/update,standard,/admin/properties,/update/json/docs
   [junit4]   2> 198389 T525 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 198391 T525 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 198391 T525 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 198393 T525 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 198393 T525 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 198394 T525 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=39, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7533346988063521]
   [junit4]   2> 198395 T525 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7ed2caf8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@410ba0d4),segFN=segments_1,generation=1}
   [junit4]   2> 198395 T525 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 198396 T525 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@17157890[collection1] main
   [junit4]   2> 198396 T525 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 198398 T525 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 198398 T525 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 198410 T525 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 198411 T525 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 198411 T525 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 198412 T525 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 198412 T525 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 198413 T525 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 198413 T525 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 198414 T526 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@17157890[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 198420 T525 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 198421 T529 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:21033/_/y collection:control_collection shard:shard1
   [junit4]   2> 198422 T495 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 198422 T495 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 198445 T529 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 198454 T529 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 198457 T521 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 198457 T529 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 198458 T529 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 198458 T522 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 198460 T495 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 198460 T495 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 198462 T533 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@231571bc name:ZooKeeperConnection Watcher:127.0.0.1:21028/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 198462 T495 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 198463 T495 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 198463 T495 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> ASYNC  NEW_CORE C105 name=collection1 org.apache.solr.core.SolrCore@6f3149d0 url=http://127.0.0.1:21033/_/y/collection1 node=127.0.0.1:21033__%2Fy C105_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:21033/_/y, core=collection1, node_name=127.0.0.1:21033__%2Fy, state=down}
   [junit4]   2> 198465 T529 C:control_collection S:shard1 C105 P21033 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:21033/_/y/collection1/
   [junit4]   2> 198465 T529 C:control_collection S:shard1 C105 P21033 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 198465 T529 C:control_collection S:shard1 C105 P21033 oasc.SyncStrategy.syncToMe http://127.0.0.1:21033/_/y/collection1/ has no replicas
   [junit4]   2> 198466 T529 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:21033/_/y/collection1/ shard1
   [junit4]   2> 198471 T495 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 198472 T521 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 198472 T533 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 198476 T529 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 198484 T521 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 198485 T522 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:21033/_/y",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 198592 T533 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 198592 T521 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 198637 T529 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:21033/_/y/collection1/ and leader is http://127.0.0.1:21033/_/y/collection1/
   [junit4]   2> 198638 T529 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:21033/_/y
   [junit4]   2> 198638 T529 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 198638 T529 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 198638 T529 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 198640 T521 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 198642 T522 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:21033/_/y",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:21033__%2Fy",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state"} current state version: 3
   [junit4]   2> 198643 T522 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:21033/_/y",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:21033__%2Fy",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 198749 T533 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 198749 T521 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 199541 T495 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/shard-1-001/cores/collection1
   [junit4]   2> 199544 T495 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/shard-1-001
   [junit4]   2> 199544 T495 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 199547 T495 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:21049
   [junit4]   2> 199547 T495 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/_/y, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/tempDir-001/jetty1, hostPort=21048, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/shard-1-001/cores, solrconfig=solrconfig.xml}
   [junit4]   2> 199547 T495 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 199548 T495 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/shard-1-001/'
   [junit4]   2> 199578 T495 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/shard-1-001/solr.xml
   [junit4]   2> 199616 T495 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/shard-1-001/cores
   [junit4]   2> 199616 T495 oasc.CoreContainer.<init> New CoreContainer 1844008169
   [junit4]   2> 199616 T495 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/shard-1-001/]
   [junit4]   2> 199617 T495 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/shard-1-001/lib
   [junit4]   2> 199617 T495 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/shard-1-001/lib).
   [junit4]   2> 199627 T495 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 199627 T495 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 199627 T495 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 199628 T495 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 199628 T495 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 199628 T495 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 199628 T495 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 199629 T495 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 199629 T495 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 199629 T495 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 199629 T495 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 199630 T495 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 199630 T495 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 199630 T495 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 199630 T495 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 199631 T495 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:21028/solr
   [junit4]   2> 199631 T495 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 199631 T495 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 199632 T495 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 199634 T545 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@76d2a91a name:ZooKeeperConnection Watcher:127.0.0.1:21028 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 199634 T495 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 199635 T495 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 199637 T495 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 199638 T548 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@33b526df name:ZooKeeperConnection Watcher:127.0.0.1:21028/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 199639 T495 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 199645 T495 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 200653 T495 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:21048__%2Fy
   [junit4]   2> 200655 T495 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:21048__%2Fy
   [junit4]   2> 200660 T495 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 200671 T495 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/shard-1-001/cores
   [junit4]   2> 200673 T495 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {shard=, loadOnStartup=true, dataDir=data/, transient=false, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/shard-1-001/cores/collection1/, coreNodeName=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/shard-1-001/cores/collection1, collection=collection1, name=collection1, schema=schema.xml, config=solrconfig.xml}
   [junit4]   2> 200673 T495 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/shard-1-001/cores/collection1/
   [junit4]   2> 200674 T495 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 200675 T549 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 200675 T549 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 200677 T521 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 200677 T549 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 200678 T522 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:21048/_/y",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:21048__%2Fy",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state"} current state version: 4
   [junit4]   2> 200679 T522 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:21048/_/y",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:21048__%2Fy",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 200679 T522 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 200680 T522 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 200682 T548 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 200682 T533 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 200682 T521 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 201677 T549 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 201678 T549 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 201679 T549 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 201679 T549 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 201680 T549 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 201680 T549 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/shard-1-001/cores/collection1/'
   [junit4]   2> 201692 T549 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 201694 T549 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 201703 T549 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 201718 T549 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 201751 T549 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 201853 T549 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 201861 T549 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 202194 T549 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 202212 T549 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 202215 T549 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 202235 T549 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 202241 T549 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 202247 T549 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 202248 T549 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 202249 T549 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 202249 T549 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 202251 T549 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 202251 T549 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 202251 T549 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 202252 T549 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 202252 T549 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 202252 T549 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@588ce5d4
   [junit4]   2> 202253 T549 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/shard-1-001/cores/collection1/data
   [junit4]   2> 202254 T549 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 202254 T549 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 202255 T549 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 202255 T549 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=40, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.15185581651979377]
   [junit4]   2> 202256 T549 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@62d6c912 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@32aef70),segFN=segments_1,generation=1}
   [junit4]   2> 202257 T549 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 202267 T549 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 202268 T549 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 202268 T549 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 202268 T549 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 202269 T549 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 202269 T549 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 202270 T549 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 202270 T549 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 202270 T549 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 202290 T549 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 202293 T549 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 202295 T549 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 202298 T549 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 202308 T549 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /replication,/update/csv,/admin/threads,/admin/mbeans,/admin/plugins,/admin/luke,/update/json,/admin/file,/admin/logging,/config,/admin/system,/get,/admin/segments,/admin/ping,/schema,/update,standard,/admin/properties,/update/json/docs
   [junit4]   2> 202310 T549 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 202312 T549 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 202312 T549 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 202314 T549 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 202314 T549 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 202315 T549 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=39, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7533346988063521]
   [junit4]   2> 202316 T549 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@62d6c912 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@32aef70),segFN=segments_1,generation=1}
   [junit4]   2> 202316 T549 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 202317 T549 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@3e3aa36a[collection1] main
   [junit4]   2> 202317 T549 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 202319 T549 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 202319 T549 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 202320 T549 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 202320 T549 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 202321 T549 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 202322 T549 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 202322 T549 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 202322 T549 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 202323 T549 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 202324 T550 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3e3aa36a[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 202324 T549 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 202325 T553 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:21048/_/y collection:collection1 shard:shard1
   [junit4]   2> 202326 T495 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 202326 T495 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 202327 T553 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 202336 T553 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 202339 T521 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 202340 T553 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 202340 T553 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C106 name=collection1 org.apache.solr.core.SolrCore@4c6c28af url=http://127.0.0.1:21048/_/y/collection1 node=127.0.0.1:21048__%2Fy C106_STATE=coll:collection1 core:collection1 props:{base_url=http://127.0.0.1:21048/_/y, core=collection1, node_name=127.0.0.1:21048__%2Fy, state=down}
   [junit4]   2> 202340 T553 C:collection1 S:shard1 C106 P21048 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:21048/_/y/collection1/
   [junit4]   2> 202340 T553 C:collection1 S:shard1 C106 P21048 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 202341 T553 C:collection1 S:shard1 C106 P21048 oasc.SyncStrategy.syncToMe http://127.0.0.1:21048/_/y/collection1/ has no replicas
   [junit4]   2> 202341 T553 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:21048/_/y/collection1/ shard1
   [junit4]   2> 202341 T522 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 202341 T553 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 202344 T533 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 202344 T548 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 202344 T521 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 202352 T522 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:21048/_/y",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 202355 T521 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 202358 T533 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 202358 T521 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 202358 T548 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 202402 T553 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:21048/_/y/collection1/ and leader is http://127.0.0.1:21048/_/y/collection1/
   [junit4]   2> 202403 T553 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:21048/_/y
   [junit4]   2> 202403 T553 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 202403 T553 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 202403 T553 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 202406 T521 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 202407 T522 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:21048/_/y",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:21048__%2Fy",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state"} current state version: 7
   [junit4]   2> 202408 T522 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:21048/_/y",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:21048__%2Fy",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 202514 T533 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 202514 T548 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 202514 T521 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 203541 T495 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 91A5A3C94AE73B0D-001/shard-2-001/cores/collection1
   [junit4]   2> 203543 T495 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in dir

[...truncated too long message...]

it4]   2> 		6	/solr/aliases.json
   [junit4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 		3	/solr/collections/c8n_1x3_commits/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent data watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		2	/solr/collections/c8n_1x3_commits/leader_elect/shard1/election/93542472712126476-core_node1-n_0000000000
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 		4	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1656108 T4011 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:37053/
   [junit4]   2> 1656108 T4011 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:37086/
   [junit4]   2> 1656109 T4011 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 1 connections to: http://127.0.0.1:37015/
   [junit4]   2> 1656109 T4011 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 10 connections to: http://127.0.0.1:36996/
   [junit4]   2> 1656109 T4011 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:36991/
   [junit4]   2> 1656110 T4011 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:37053/
   [junit4]   2> 1656110 T4011 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:37086/
   [junit4]   2> 1656110 T4011 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:37015/
   [junit4]   2> 1656110 T4011 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:36996/
   [junit4]   2> 1656110 T4011 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:36991/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderInitiatedRecoveryOnCommitTest -Dtests.method=test -Dtests.seed=91A5A3C94AE73B0D -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ar_AE -Dtests.timezone=Europe/Zagreb -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   83.2s J1 | LeaderInitiatedRecoveryOnCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37053/c8n_1x3_commits_shard1_replica2
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([91A5A3C94AE73B0D:19F19C13E41B56F5]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
   [junit4]    > 	... 48 more
   [junit4]   2> 1656127 T4011 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 91A5A3C94AE73B0D-001
   [junit4]   2> 83253 T4010 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=false,coord=no): {}, locale=ar_AE, timezone=Europe/Zagreb
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=139938040,total=324534272
   [junit4]   2> NOTE: All tests run in this JVM: [TestReqParamsAPI, TestSolrQueryParserDefaultOperatorResource, TestConfigSets, TestCollapseQParserPlugin, BasicDistributedZkTest, UpdateParamsTest, TestNamedUpdateProcessors, TestSchemaVersionResource, EchoParamsTest, AsyncMigrateRouteKeyTest, ZkCLITest, ActionThrottleTest, DistanceUnitsTest, BadCopyFieldTest, ReplicationFactorTest, TestChildDocTransformer, TestDefaultSearchFieldResource, IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestDynamicFieldCollectionResource, FullSolrCloudDistribCmdsTest, TestElisionMultitermQuery, TestDynamicFieldResource, TestUpdate, VMParamsZkACLAndCredentialsProvidersTest, TestGroupingSearch, DateMathParserTest, TestExactStatsCache, TestSolr4Spatial, SimpleMLTQParserTest, RequestHandlersTest, ClusterStateUpdateTest, TestFiltering, TimeZoneUtilsTest, TestSweetSpotSimilarityFactory, DisMaxRequestHandlerTest, TestRandomMergePolicy, HdfsDirectoryTest, TermsComponentTest, DeleteInactiveReplicaTest, TestSolr4Spatial2, UniqFieldsUpdateProcessorFactoryTest, OutputWriterTest, TestSurroundQueryParser, TestAnalyzedSuggestions, SignatureUpdateProcessorFactoryTest, CopyFieldTest, TestManagedSchemaFieldResource, TestLRUCache, TestTrackingShardHandlerFactory, TestHighlightDedupGrouping, TestClassNameShortening, SaslZkACLProviderTest, TestNonDefinedSimilarityFactory, OutOfBoxZkACLAndCredentialsProvidersTest, TestUniqueKeyFieldResource, TestManagedSynonymFilterFactory, OverseerStatusTest, LeaderFailoverAfterPartitionTest, TestReplicationHandler, LeaderInitiatedRecoveryOnCommitTest]
   [junit4] Completed on J1 in 83.80s, 1 test, 1 error <<< FAILURES!

[...truncated 730 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 476 suites, 1896 tests, 3 errors, 62 ignored (22 assumptions)

Total time: 56 minutes 52 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 216907993 bytes
Compression is 0.0%
Took 45 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2834 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2834/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:36997/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:36997/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([71BA1E4371045C9C:F9EE2199DFF83164]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:59006/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:59006/collection1
	at __randomizedtesting.SeedInfo.seed([71BA1E4371045C9C:F9EE2199DFF83164]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:59006/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:32535/dh/repfacttest_c8n_1x3_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:32535/dh/repfacttest_c8n_1x3_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([71BA1E4371045C9C:F9EE2199DFF83164]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:32535/dh/repfacttest_c8n_1x3_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 53 more




Build Log:
[...truncated 9364 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/init-core-data-001
   [junit4]   2> 189984 T812 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 190013 T812 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 190021 T813 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 190426 T812 oasc.ZkTestServer.run start zk server on port:36981
   [junit4]   2> 190521 T812 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 190680 T812 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 190797 T820 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1d4cf0c0 name:ZooKeeperConnection Watcher:127.0.0.1:36981 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 190798 T812 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 190799 T812 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 190806 T812 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 190890 T812 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 190891 T812 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 190893 T823 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7725f101 name:ZooKeeperConnection Watcher:127.0.0.1:36981/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 190893 T812 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 190894 T812 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 190907 T812 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 190911 T812 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 190913 T812 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 190916 T812 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 190919 T812 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 190919 T812 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 190924 T812 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 190924 T812 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 190927 T812 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 190928 T812 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 190931 T812 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 190931 T812 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 190934 T812 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 190935 T812 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 190938 T812 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 190938 T812 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 190941 T812 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 190942 T812 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 190945 T812 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 190945 T812 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 190948 T812 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 190948 T812 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 190951 T812 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 190952 T812 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 190955 T812 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 190955 T812 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 191962 T812 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/control-001/cores/collection1
   [junit4]   2> 191966 T812 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 191969 T812 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:36987
   [junit4]   2> 191970 T812 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=36986, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/tempDir-001/control/data, hostContext=/}
   [junit4]   2> 191970 T812 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 191970 T812 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/control-001/'
   [junit4]   2> 192002 T812 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/control-001/solr.xml
   [junit4]   2> 192044 T812 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/control-001/cores
   [junit4]   2> 192044 T812 oasc.CoreContainer.<init> New CoreContainer 916278383
   [junit4]   2> 192045 T812 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/control-001/]
   [junit4]   2> 192045 T812 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/control-001/lib
   [junit4]   2> 192045 T812 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/control-001/lib).
   [junit4]   2> 192055 T812 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 192055 T812 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 192056 T812 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 192056 T812 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 192056 T812 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 192056 T812 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 192057 T812 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 192057 T812 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 192057 T812 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 192057 T812 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 192058 T812 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 192058 T812 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 192059 T812 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 192059 T812 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 192059 T812 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 192060 T812 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:36981/solr
   [junit4]   2> 192098 T812 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 192098 T812 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 192099 T812 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 192101 T835 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4e18e6f4 name:ZooKeeperConnection Watcher:127.0.0.1:36981 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 192101 T812 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 192102 T812 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 192117 T812 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 192119 T838 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2d0c60d4 name:ZooKeeperConnection Watcher:127.0.0.1:36981/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 192120 T812 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 192136 T812 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 192153 T812 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 192161 T812 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 192164 T812 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 192168 T812 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 192191 T812 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 192193 T812 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 192196 T812 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 192197 T812 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36986_
   [junit4]   2> 192204 T812 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:36986_
   [junit4]   2> 192207 T812 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 192209 T812 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 192212 T812 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 192229 T812 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:36986_
   [junit4]   2> 192230 T812 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 192232 T812 oasc.Overseer.start Overseer (id=93541442786492419-127.0.0.1:36986_-n_0000000000) starting
   [junit4]   2> 192249 T812 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 192279 T812 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 192297 T840 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 192297 T812 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 192328 T839 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 192332 T812 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/control-001/cores
   [junit4]   2> 192348 T812 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=control_collection, coreNodeName=, config=solrconfig.xml, transient=false, shard=, dataDir=data/, schema=schema.xml, loadOnStartup=true, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/control-001/cores/collection1/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/control-001/cores/collection1, name=collection1}
   [junit4]   2> 192349 T812 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/control-001/cores/collection1/
   [junit4]   2> 192349 T812 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 192350 T842 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 192350 T842 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 192366 T838 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 192366 T842 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 192372 T839 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:36986_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:36986",
   [junit4]   2> 	  "operation":"state"} current state version: 0
   [junit4]   2> 192414 T839 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:36986_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:36986",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 192420 T839 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 192455 T839 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 192471 T838 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 193369 T842 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 193369 T842 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 193371 T842 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 193371 T842 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 193373 T842 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 193373 T842 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/control-001/cores/collection1/'
   [junit4]   2> 193413 T842 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 193429 T842 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 193450 T842 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 193471 T842 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 193528 T842 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 193530 T842 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 193539 T842 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 193832 T842 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 193845 T842 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 193847 T842 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 193866 T842 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 193870 T842 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 193874 T842 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 193875 T842 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 193875 T842 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 193876 T842 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 193877 T842 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 193877 T842 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 193877 T842 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 193878 T842 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 193878 T842 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/control-001/cores/collection1/data/
   [junit4]   2> 193878 T842 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4955aabe
   [junit4]   2> 193889 T842 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/control-001/cores/collection1/data
   [junit4]   2> 193889 T842 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/control-001/cores/collection1/data/index/
   [junit4]   2> 193889 T842 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 193890 T842 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/control-001/cores/collection1/data/index
   [junit4]   2> 193890 T842 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=32, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7067390937442912]
   [junit4]   2> 193891 T842 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@ba4c085 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6d5b61bb),segFN=segments_1,generation=1}
   [junit4]   2> 193891 T842 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 193899 T842 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 193945 T842 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 193945 T842 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 193945 T842 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 193946 T842 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 193946 T842 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 193946 T842 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 193947 T842 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 193947 T842 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 193960 T842 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 193967 T842 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 193969 T842 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 193971 T842 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 193981 T842 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: standard,/update/csv,/update,/admin/luke,/admin/properties,/config,/admin/ping,/admin/segments,/admin/mbeans,/replication,/admin/file,/admin/threads,/admin/system,/schema,/update/json,/admin/logging,/get,/update/json/docs,/admin/plugins
   [junit4]   2> 193983 T842 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 193984 T842 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 193984 T842 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 193986 T842 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 193986 T842 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 193987 T842 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=21, maxMergeAtOnceExplicit=50, maxMergedSegmentMB=24.3388671875, floorSegmentMB=1.8701171875, forceMergeDeletesPctAllowed=28.58885003317121, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8949881822224737
   [junit4]   2> 193988 T842 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@ba4c085 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6d5b61bb),segFN=segments_1,generation=1}
   [junit4]   2> 193988 T842 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 193988 T842 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@4e4f880c[collection1] main
   [junit4]   2> 193989 T842 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 193990 T842 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 193990 T842 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 193999 T842 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 193999 T842 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 193999 T842 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 194000 T842 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 194001 T842 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 194001 T842 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 194001 T842 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 194004 T843 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4e4f880c[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 194009 T842 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 194010 T846 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:36986 collection:control_collection shard:shard1
   [junit4]   2> 194011 T812 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 194011 T812 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 194042 T846 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 194051 T846 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 194054 T838 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 194055 T846 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 194055 T846 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C86 name=collection1 org.apache.solr.core.SolrCore@93d77f3 url=http://127.0.0.1:36986/collection1 node=127.0.0.1:36986_ C86_STATE=coll:control_collection core:collection1 props:{state=down, core=collection1, node_name=127.0.0.1:36986_, base_url=http://127.0.0.1:36986}
   [junit4]   2> 194056 T846 C:control_collection S:shard1 C86 P36986 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:36986/collection1/
   [junit4]   2> 194056 T839 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 194056 T846 C:control_collection S:shard1 C86 P36986 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 194057 T846 C:control_collection S:shard1 C86 P36986 oasc.SyncStrategy.syncToMe http://127.0.0.1:36986/collection1/ has no replicas
   [junit4]   2> 194057 T846 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:36986/collection1/ shard1
   [junit4]   2> 194066 T812 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 194067 T812 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 194069 T850 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@44ea2bc4 name:ZooKeeperConnection Watcher:127.0.0.1:36981/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 194069 T838 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 194069 T812 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 194071 T812 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 194071 T812 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 194073 T846 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 194080 T812 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 194080 T812 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 194081 T812 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 194081 T812 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 194082 T838 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 194084 T853 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@60cb0f63 name:ZooKeeperConnection Watcher:127.0.0.1:36981/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 194084 T812 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 194084 T839 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:36986",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 194085 T812 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 194090 T839 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 194090 T839 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 194092 T838 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 194092 T850 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 194099 T838 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 194101 T839 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 194104 T838 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 194104 T850 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 194134 T846 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:36986/collection1/ and leader is http://127.0.0.1:36986/collection1/
   [junit4]   2> 194135 T846 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:36986
   [junit4]   2> 194135 T846 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 194136 T846 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 194136 T846 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 194152 T838 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 194154 T839 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:36986_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:36986",
   [junit4]   2> 	  "operation":"state"} current state version: 4
   [junit4]   2> 194155 T839 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:36986_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:36986",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 194261 T838 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 194261 T850 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 195380 T812 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/shard-1-001/cores/collection1
   [junit4]   2> 195383 T812 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/shard-1-001
   [junit4]   2> 195383 T812 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 195385 T812 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:36998
   [junit4]   2> 195385 T812 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/shard-1-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/tempDir-001/jetty1, hostContext=/, solrconfig=solrconfig.xml, hostPort=36997}
   [junit4]   2> 195386 T812 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 195386 T812 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/shard-1-001/'
   [junit4]   2> 195423 T812 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/shard-1-001/solr.xml
   [junit4]   2> 195467 T812 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/shard-1-001/cores
   [junit4]   2> 195468 T812 oasc.CoreContainer.<init> New CoreContainer 273237318
   [junit4]   2> 195468 T812 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/shard-1-001/]
   [junit4]   2> 195469 T812 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/shard-1-001/lib
   [junit4]   2> 195469 T812 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/shard-1-001/lib).
   [junit4]   2> 195479 T812 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 195479 T812 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 195479 T812 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 195480 T812 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 195480 T812 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 195480 T812 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 195481 T812 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 195481 T812 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 195481 T812 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 195481 T812 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 195482 T812 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 195482 T812 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 195482 T812 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 195483 T812 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 195483 T812 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 195483 T812 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:36981/solr
   [junit4]   2> 195484 T812 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 195484 T812 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 195485 T812 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 195487 T865 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5b4f2995 name:ZooKeeperConnection Watcher:127.0.0.1:36981 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 195487 T812 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 195488 T812 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 195490 T812 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 195492 T868 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@41be96aa name:ZooKeeperConnection Watcher:127.0.0.1:36981/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 195492 T812 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 195499 T812 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 196507 T812 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36997_
   [junit4]   2> 196509 T812 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:36997_
   [junit4]   2> 196513 T812 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 196523 T812 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/shard-1-001/cores
   [junit4]   2> 196525 T812 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, collection=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/shard-1-001/cores/collection1, transient=false, coreNodeName=, shard=, loadOnStartup=true, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/shard-1-001/cores/collection1/, name=collection1, schema=schema.xml, dataDir=data/}
   [junit4]   2> 196525 T812 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/shard-1-001/cores/collection1/
   [junit4]   2> 196526 T812 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 196529 T869 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 196529 T869 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 196531 T838 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 196532 T869 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 196532 T869 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 196533 T839 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:36997_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:36997",
   [junit4]   2> 	  "operation":"state"} current state version: 5
   [junit4]   2> 196533 T869 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 196534 T839 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:36997_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:36997",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 196534 T839 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 196534 T839 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 196536 T839 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 196542 T869 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 196542 T869 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 196542 T869 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 196542 T869 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 196543 T869 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 196543 T869 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 196544 T869 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 196544 T869 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/shard-1-001/cores/collection1/'
   [junit4]   2> 196557 T869 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 196560 T869 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 196571 T869 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 196589 T869 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 196629 T869 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 196631 T869 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 196640 T869 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 196932 T869 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 196945 T869 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 196947 T869 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 196964 T869 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 196968 T869 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 196971 T869 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 196973 T869 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 196973 T869 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 196973 T869 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 196974 T869 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 196975 T869 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 196975 T869 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 196975 T869 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 196975 T869 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 196976 T869 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4955aabe
   [junit4]   2> 196977 T869 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/shard-1-001/cores/collection1/data
   [junit4]   2> 196977 T869 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 196977 T869 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 196977 T869 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 71BA1E4371045C9C-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 196978 T869 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=32, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7067390937442912]
   [junit4]   2> 196979 T869 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@28e68a2f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@377a28a3),segFN=segments_1,generation=1}
   [junit4]   2> 196979 T869 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 196989 T869 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 196989 T869 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 196989 T869 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 196990 T869 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 196990 T869 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 196990 T869 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 196991 T869 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 196991 T869 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 196991 T869 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 197003 T869 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 197006 T869 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 197009 T869 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 197011 T869 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 197028 T869 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: standard,/update/csv,/update,/admin/luke,/admin/properties,/config,/admin/ping,/admin/segments,/admin/mbeans,/replication,/admin/file,/admin/threads,/admin/system,/schema,/update/json,/admin/logging,/get,/update/json/docs,/admin/plugins
   [junit4]   2> 197030 T869 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 197031 T869 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 197031 T869 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 197032 T869 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 197033 T869 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 197034 T869 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=21, maxMergeAtOnceExplicit=50, maxMergedSegmentMB=24.3388671875, floorSegmentMB=1.8701171875, forceMergeDeletesPctAllowed=28.58885003317121, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8949881822224737
   [junit4]   2> 197035 T869 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@28e68a2f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@377a28a3),segFN=segments_1,generation=1}
   [junit4]   2> 197035 T869 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 197035 T869 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@12c1bb86[collection1] main
   [junit4]   2> 197035 T869 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 197037 T869 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 197037 T869 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 197038 T869 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 197038 T869 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 197038 T869 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 197039 T869 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 197039 T869 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 197039 T869 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 197039 T869 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 197042 T870 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@12c1bb86[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 197043 T869 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 197043 T873 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:36997 collection:collection1 shard:shard1
   [junit4]   2> 197044 T812 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 197044 T812 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 197044 T873 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 197051 T873 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 197053 T838 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 197054 T873 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 197054 T873 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C87 name=collection1 org.apache.solr.core.SolrCore@2ff9e385 url=http://127.0.0.1:36997/collection1 node=127.0.0.1:36997_ C87_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, node_name=127.0.0.1:36997_, base_url=http://127.0.0.1:36997}
   [junit4]   2> 197054 T873 C:collection1 S:shard1 C87 P36997 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:36997/collection1/
   [junit4]   2> 197054 T873 C:collection1 S:shard1 C87 P36997 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 197054 T839 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 197054 T873 C:collection1 S:shard1 C87 P36997 oasc.SyncStrategy.syncToMe http://127.0.0.1:36997/collection1/ has no replicas
   [junit4]   2> 197055 T873 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:36997/collection1/ shard1
   [junit4]   2> 197055 T873 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 197056 T839 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 197057 T868 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 197058 T868 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 197062 T839 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:36997",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 197064 T839 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 197064 T868 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 197065 T868 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 197068 T838 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 197112 T873 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:36997/collection1/ and leader is http://127.0.0.1:36997/collection1/
   [junit4]   2> 197113 T873 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:36997
   [junit4]   2> 197113 T873 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 197113 T873 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 197113 T873 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 197115 T838 oasc.DistributedQueue$Latch

[...truncated too long message...]

146135 T2072 C:control_collection S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:58997 58997
   [junit4]   2> 1146235 T2073 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:58997 58997
   [junit4]   2> 1146236 T2073 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 		2	/solr/collections/collection1/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1146236 T2072 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:59006/
   [junit4]   2> 1146237 T2072 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:59000/
   [junit4]   2> 1146237 T2072 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:59010/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=71BA1E4371045C9C -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=th_TH_TH_#u-nu-thai -Dtests.timezone=Asia/Dacca -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   67.6s J0 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:59006/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([71BA1E4371045C9C:F9EE2199DFF83164]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:59006/collection1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 49 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
   [junit4]    > 	... 54 more
   [junit4]   2> 1146257 T2072 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest 71BA1E4371045C9C-001
   [junit4]   2> 67630 T2071 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {a_t=PostingsFormat(name=Memory doPackFST= true), id=Lucene50(blocksize=128), timestamp=PostingsFormat(name=Memory doPackFST= true), range_facet_l=PostingsFormat(name=Memory doPackFST= true), intDefault=PostingsFormat(name=Memory doPackFST= true), text=BlockTreeOrds(blocksize=128), _version_=PostingsFormat(name=Memory doPackFST= true), multiDefault=FST50}, docValues:{timestamp=DocValuesFormat(name=Lucene50)}, sim=DefaultSimilarity, locale=th_TH_TH_#u-nu-thai, timezone=Asia/Dacca
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=184124640,total=270008320
   [junit4]   2> NOTE: All tests run in this JVM: [DistributedMLTComponentTest, TestPivotHelperCode, DistributedQueueTest, ZkNodePropsTest, HttpPartitionTest, DistributedFacetPivotLargeTest, StressHdfsTest, AliasIntegrationTest, TestWriterPerf, TestHighFrequencyDictionaryFactory, DistributedDebugComponentTest, SchemaVersionSpecificBehaviorTest, TestMissingGroups, TestSolrQueryParser, DocValuesMissingTest, TestDistributedGrouping, QueryResultKeyTest, TestManagedResource, TestSolrConfigHandler, TestRandomFaceting, TestQuerySenderNoQuery, CacheHeaderTest, DirectUpdateHandlerOptimizeTest, CoreMergeIndexesAdminHandlerTest, TestCSVLoader, TestFuzzyAnalyzedSuggestions, RequiredFieldsTest, TestSerializedLuceneMatchVersion, TestRequestStatusCollectionAPI, TestManagedStopFilterFactory, TestChildDocTransformer, TermsComponentTest, TestExactSharedStatsCache, UnloadDistributedZkTest, TestOmitPositions, CircularListTest, TestFieldResource, SignatureUpdateProcessorFactoryTest, TestFaceting, CloudMLTQParserTest, SolrCoreCheckLockOnStartupTest, StandardRequestHandlerTest, QueryParsingTest, TestDFRSimilarityFactory, HdfsLockFactoryTest, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J0 in 68.17s, 1 test, 1 error <<< FAILURES!

[...truncated 805 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 476 suites, 1896 tests, 3 errors, 62 ignored (22 assumptions)

Total time: 54 minutes 12 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 206604595 bytes
Compression is 0.0%
Took 40 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2833 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2833/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:48343/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48343/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([F890540B8D1E93B0:70C46BD123E2FE48]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([F890540B8D1E93B0:70C46BD123E2FE48]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([F890540B8D1E93B0:70C46BD123E2FE48]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9788 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/init-core-data-001
   [junit4]   2> 928960 T4152 [] [] [] []  oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /vbje/t
   [junit4]   2> 928967 T4152 [] [] [] []  oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 928967 T4153 [] [] [] []  oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 929067 T4152 [] [] [] []  oasc.ZkTestServer.run start zk server on port:61260
   [junit4]   2> 929068 T4152 [] [] [] []  oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 929068 T4152 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 929072 T4160 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@708a7810 name:ZooKeeperConnection Watcher:127.0.0.1:61260 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 929072 T4152 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 929073 T4152 [] [] [] []  oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 929073 T4152 [] [] [] []  oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 929075 T4152 [] [] [] []  oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 929076 T4152 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 929077 T4163 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5b347a56 name:ZooKeeperConnection Watcher:127.0.0.1:61260/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 929078 T4152 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 929078 T4152 [] [] [] []  oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 929079 T4152 [] [] [] []  oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 929080 T4152 [] [] [] []  oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 929082 T4152 [] [] [] []  oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 929083 T4152 [] [] [] []  oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 929085 T4152 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 929086 T4152 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 929088 T4152 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 929089 T4152 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 929191 T4152 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 929191 T4152 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 929193 T4152 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 929194 T4152 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 929195 T4152 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 929196 T4152 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 929198 T4152 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 929198 T4152 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 929200 T4152 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 929200 T4152 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 929202 T4152 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 929203 T4152 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 929204 T4152 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 929205 T4152 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 929206 T4152 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 929207 T4152 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 929209 T4152 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 929209 T4152 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 930402 T4152 [] [] [] []  oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/control-001/cores/collection1
   [junit4]   2> 930406 T4152 [] [] [] []  oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 930410 T4152 [] [] [] []  oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:61264
   [junit4]   2> 930410 T4152 [] [] [] []  oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=61263, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/control-001/cores, hostContext=/vbje/t, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/tempDir-001/control/data}
   [junit4]   2> 930411 T4152 [] [] [] []  oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 930411 T4152 [] [] [] []  oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/control-001/'
   [junit4]   2> 930441 T4152 [] [] [] []  oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/control-001/solr.xml
   [junit4]   2> 930475 T4152 [] [] [] []  oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/control-001/cores
   [junit4]   2> 930476 T4152 [] [] [] []  oasc.CoreContainer.<init> New CoreContainer 709235109
   [junit4]   2> 930476 T4152 [] [] [] []  oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/control-001/]
   [junit4]   2> 930476 T4152 [] [] [] []  oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/control-001/lib
   [junit4]   2> 930477 T4152 [] [] [] []  oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/control-001/lib).
   [junit4]   2> 930497 T4152 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 930497 T4152 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 930497 T4152 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 930497 T4152 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 930498 T4152 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 930498 T4152 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 930498 T4152 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 930498 T4152 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 930499 T4152 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 930499 T4152 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 930499 T4152 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 930499 T4152 [] [] [] []  oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 930500 T4152 [] [] [] []  oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 930500 T4152 [] [] [] []  oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 930500 T4152 [] [] [] []  oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 930500 T4152 [] [] [] []  oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:61260/solr
   [junit4]   2> 930501 T4152 [] [] [] []  oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 930501 T4152 [] [] [] []  oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 930502 T4152 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 930503 T4175 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@610b776c name:ZooKeeperConnection Watcher:127.0.0.1:61260 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 930504 T4152 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 930504 T4152 [] [] [] []  oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 930506 T4152 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 930507 T4178 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@32ce1ca2 name:ZooKeeperConnection Watcher:127.0.0.1:61260/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 930507 T4152 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 930508 T4152 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 930509 T4152 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 930511 T4152 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 930512 T4152 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 930513 T4152 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 930515 T4152 [] [] [] []  oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 930516 T4152 [] [] [] []  oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 930517 T4152 [] [] [] []  oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 930517 T4152 [] [] [] []  oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:61263_vbje%2Ft
   [junit4]   2> 930518 T4152 [] [] [] []  oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:61263_vbje%2Ft
   [junit4]   2> 930519 T4152 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 930520 T4152 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 930521 T4152 [] [] [] []  oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 930522 T4152 [] [] [] []  oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:61263_vbje%2Ft
   [junit4]   2> 930522 T4152 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 930523 T4152 [] [] [] []  oasc.Overseer.start Overseer (id=93540542416945155-127.0.0.1:61263_vbje%2Ft-n_0000000000) starting
   [junit4]   2> 930524 T4152 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 930528 T4152 [] [] [] []  oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 930528 T4180 [] [] [] []  oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 930528 T4179 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 930528 T4152 [] [] [] []  oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 930532 T4152 [] [] [] []  oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/control-001/cores
   [junit4]   2> 930533 T4152 [] [] [] []  oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, transient=false, coreNodeName=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/control-001/cores/collection1, collection=control_collection, config=solrconfig.xml, loadOnStartup=true, shard=, schema=schema.xml, dataDir=data/, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/control-001/cores/collection1/}
   [junit4]   2> 930533 T4152 [] [] [] []  oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/control-001/cores/collection1/
   [junit4]   2> 930534 T4152 [] [] [] []  oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 930534 T4182 [control_collection] [] [] [collection1]  oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 930534 T4182 [control_collection] [] [] [collection1]  oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 930535 T4182 [control_collection] [] [] [collection1]  oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 930535 T4178 [] [] [] []  oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 930536 T4179 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:61263_vbje%2Ft",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:61263/vbje/t",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1"} current state version: 0
   [junit4]   2> 930536 T4179 [] [] [] []  oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:61263_vbje%2Ft",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:61263/vbje/t",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 930536 T4179 [] [] [] []  oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 930537 T4179 [] [] [] []  oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 930537 T4178 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 931535 T4182 [control_collection] [] [] [collection1]  oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 931536 T4182 [control_collection] [] [] [collection1]  oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 931536 T4182 [control_collection] [] [] [collection1]  oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 931537 T4182 [control_collection] [] [] [collection1]  oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 931537 T4182 [control_collection] [] [] [collection1]  oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 931537 T4182 [control_collection] [] [] [collection1]  oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/control-001/cores/collection1/'
   [junit4]   2> 931554 T4182 [control_collection] [] [] [collection1]  oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 931557 T4182 [control_collection] [] [] [collection1]  oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 931567 T4182 [control_collection] [] [] [collection1]  oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 931589 T4182 [control_collection] [] [] [collection1]  oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 931638 T4182 [control_collection] [] [] [collection1]  oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 931739 T4182 [control_collection] [] [] [collection1]  oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 931750 T4182 [control_collection] [] [] [collection1]  oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 932092 T4182 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 932104 T4182 [control_collection] [] [] [collection1]  oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 932106 T4182 [control_collection] [] [] [collection1]  oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 932113 T4182 [control_collection] [] [] [collection1]  oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 932116 T4182 [control_collection] [] [] [collection1]  oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 932119 T4182 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 932120 T4182 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 932120 T4182 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 932120 T4182 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 932120 T4182 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 932121 T4182 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 932121 T4182 [control_collection] [] [] [collection1]  oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 932121 T4182 [control_collection] [] [] [collection1]  oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 932121 T4182 [control_collection] [] [] [collection1]  oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/control-001/cores/collection1/data/
   [junit4]   2> 932122 T4182 [control_collection] [] [] [collection1]  oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1f0fee7c
   [junit4]   2> 932122 T4182 [control_collection] [] [] [collection1]  oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/control-001/cores/collection1/data
   [junit4]   2> 932122 T4182 [control_collection] [] [] [collection1]  oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/control-001/cores/collection1/data/index/
   [junit4]   2> 932123 T4182 [control_collection] [] [] [collection1]  oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 932123 T4182 [control_collection] [] [] [collection1]  oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/control-001/cores/collection1/data/index
   [junit4]   2> 932123 T4182 [control_collection] [] [] [collection1]  oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=17, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=18.5205078125, floorSegmentMB=0.8837890625, forceMergeDeletesPctAllowed=27.25970516083252, segmentsPerTier=10.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 932124 T4182 [control_collection] [] [] [collection1]  oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2a76c2f6 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5af9ba41),segFN=segments_1,generation=1}
   [junit4]   2> 932124 T4182 [control_collection] [] [] [collection1]  oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 932128 T4182 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 932129 T4182 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 932129 T4182 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 932129 T4182 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 932129 T4182 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 932129 T4182 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 932130 T4182 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 932130 T4182 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 932130 T4182 [control_collection] [] [] [collection1]  oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 932142 T4182 [control_collection] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 932144 T4182 [control_collection] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 932145 T4182 [control_collection] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 932147 T4182 [control_collection] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 932151 T4182 [control_collection] [] [] [collection1]  oasc.RequestHandlers.initHandlersFromConfig Registered paths: /get,/schema,/update/json/docs,/admin/luke,/replication,/admin/file,/admin/segments,/admin/plugins,/config,/admin/ping,/admin/threads,/update/json,/admin/logging,/admin/mbeans,/admin/system,standard,/update/csv,/update,/admin/properties
   [junit4]   2> 932152 T4182 [control_collection] [] [] [collection1]  oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 932153 T4182 [control_collection] [] [] [collection1]  oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 932153 T4182 [control_collection] [] [] [collection1]  oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 932154 T4182 [control_collection] [] [] [collection1]  oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 932154 T4182 [control_collection] [] [] [collection1]  oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 932155 T4182 [control_collection] [] [] [collection1]  oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=42, maxMergeAtOnceExplicit=33, maxMergedSegmentMB=79.6669921875, floorSegmentMB=1.30078125, forceMergeDeletesPctAllowed=11.071207434441273, segmentsPerTier=27.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.47216918996256496
   [junit4]   2> 932155 T4182 [control_collection] [] [] [collection1]  oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2a76c2f6 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5af9ba41),segFN=segments_1,generation=1}
   [junit4]   2> 932156 T4182 [control_collection] [] [] [collection1]  oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 932156 T4182 [control_collection] [] [] [collection1]  oass.SolrIndexSearcher.<init> Opening Searcher@23dabefb[collection1] main
   [junit4]   2> 932156 T4182 [control_collection] [] [] [collection1]  oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 932157 T4182 [control_collection] [] [] [collection1]  oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 932157 T4182 [control_collection] [] [] [collection1]  oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 932157 T4182 [control_collection] [] [] [collection1]  oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 932157 T4182 [control_collection] [] [] [collection1]  oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 932158 T4182 [control_collection] [] [] [collection1]  oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 932158 T4182 [control_collection] [] [] [collection1]  oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 932158 T4182 [control_collection] [] [] [collection1]  oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 932158 T4182 [control_collection] [] [] [collection1]  oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 932159 T4182 [control_collection] [] [] [collection1]  oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 932159 T4183 [control_collection] [] [] [collection1]  oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@23dabefb[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 932160 T4182 [control_collection] [] [] [collection1]  oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 932161 T4186 [control_collection] [shard1] [] []  oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:61263/vbje/t collection:control_collection shard:shard1
   [junit4]   2> 932161 T4152 [] [] [] []  oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 932161 T4152 [] [] [] []  oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 932161 T4186 [control_collection] [shard1] [] []  oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 932165 T4152 [] [] [] []  oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 932165 T4152 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 932165 T4186 [control_collection] [shard1] [] []  oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 932167 T4190 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3a7f34c2 name:ZooKeeperConnection Watcher:127.0.0.1:61260/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 932167 T4152 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 932167 T4178 [] [] [] []  oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 932168 T4152 [] [] [] []  oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 932168 T4186 [control_collection] [shard1] [] []  oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 932168 T4186 [control_collection] [shard1] [] []  oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 932168 T4152 [] [] [] []  oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> ASYNC  NEW_CORE C2349 name=collection1 org.apache.solr.core.SolrCore@52df9fd8 url=http://127.0.0.1:61263/vbje/t/collection1 node=127.0.0.1:61263_vbje%2Ft C2349_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:61263_vbje%2Ft, base_url=http://127.0.0.1:61263/vbje/t, state=down, core=collection1}
   [junit4]   2> 932168 T4186 [control_collection] [shard1] [] []  C2349 P61263 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:61263/vbje/t/collection1/
   [junit4]   2> 932168 T4179 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 932169 T4186 [control_collection] [shard1] [] []  C2349 P61263 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 932169 T4186 [control_collection] [shard1] [] []  C2349 P61263 oasc.SyncStrategy.syncToMe http://127.0.0.1:61263/vbje/t/collection1/ has no replicas
   [junit4]   2> 932170 T4186 [control_collection] [shard1] [] []  oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:61263/vbje/t/collection1/ shard1
   [junit4]   2> 932170 T4186 [control_collection] [shard1] [] []  oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 932170 T4178 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 932170 T4190 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 932172 T4152 [] [] [] []  oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 932175 T4178 [] [] [] []  oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 932175 T4179 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:61263/vbje/t",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 932278 T4178 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 932278 T4190 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 932327 T4186 [control_collection] [shard1] [] []  oasc.ZkController.register We are http://127.0.0.1:61263/vbje/t/collection1/ and leader is http://127.0.0.1:61263/vbje/t/collection1/
   [junit4]   2> 932328 T4186 [control_collection] [shard1] [] []  oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:61263/vbje/t
   [junit4]   2> 932328 T4186 [control_collection] [shard1] [] []  oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 932328 T4186 [control_collection] [shard1] [] [collection1]  oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 932328 T4186 [control_collection] [shard1] [] [collection1]  oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 932329 T4178 [] [] [] []  oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 932330 T4179 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:61263_vbje%2Ft",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:61263/vbje/t",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1"} current state version: 3
   [junit4]   2> 932331 T4179 [] [] [] []  oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:61263_vbje%2Ft",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:61263/vbje/t",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 932433 T4190 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 932433 T4178 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 933337 T4152 [] [] [] []  oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/shard-1-001/cores/collection1
   [junit4]   2> 933339 T4152 [] [] [] []  oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/shard-1-001
   [junit4]   2> 933339 T4152 [] [] [] []  oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 933342 T4152 [] [] [] []  oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:61269
   [junit4]   2> 933342 T4152 [] [] [] []  oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/vbje/t, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/shard-1-001/cores, solrconfig=solrconfig.xml, hostPort=61268, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/tempDir-001/jetty1}
   [junit4]   2> 933343 T4152 [] [] [] []  oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 933343 T4152 [] [] [] []  oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/shard-1-001/'
   [junit4]   2> 933372 T4152 [] [] [] []  oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/shard-1-001/solr.xml
   [junit4]   2> 933407 T4152 [] [] [] []  oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/shard-1-001/cores
   [junit4]   2> 933408 T4152 [] [] [] []  oasc.CoreContainer.<init> New CoreContainer 278696851
   [junit4]   2> 933408 T4152 [] [] [] []  oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/shard-1-001/]
   [junit4]   2> 933408 T4152 [] [] [] []  oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/shard-1-001/lib
   [junit4]   2> 933409 T4152 [] [] [] []  oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/shard-1-001/lib).
   [junit4]   2> 933417 T4152 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 933418 T4152 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 933418 T4152 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 933418 T4152 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 933418 T4152 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 933419 T4152 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 933419 T4152 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 933419 T4152 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 933419 T4152 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 933419 T4152 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 933420 T4152 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 933420 T4152 [] [] [] []  oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 933420 T4152 [] [] [] []  oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 933421 T4152 [] [] [] []  oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 933421 T4152 [] [] [] []  oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 933421 T4152 [] [] [] []  oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:61260/solr
   [junit4]   2> 933421 T4152 [] [] [] []  oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 933422 T4152 [] [] [] []  oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 933422 T4152 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 933424 T4202 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@585023e1 name:ZooKeeperConnection Watcher:127.0.0.1:61260 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 933424 T4152 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 933425 T4152 [] [] [] []  oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 933427 T4152 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 933428 T4205 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@a6f155e name:ZooKeeperConnection Watcher:127.0.0.1:61260/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 933428 T4152 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 933432 T4152 [] [] [] []  oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 934435 T4152 [] [] [] []  oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:61268_vbje%2Ft
   [junit4]   2> 934436 T4152 [] [] [] []  oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:61268_vbje%2Ft
   [junit4]   2> 934439 T4152 [] [] [] []  oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 934441 T4152 [] [] [] []  oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/shard-1-001/cores
   [junit4]   2> 934442 T4152 [] [] [] []  oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {coreNodeName=, name=collection1, shard=, transient=false, loadOnStartup=true, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/shard-1-001/cores/collection1/, config=solrconfig.xml, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/shard-1-001/cores/collection1, dataDir=data/, collection=collection1}
   [junit4]   2> 934443 T4152 [] [] [] []  oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/shard-1-001/cores/collection1/
   [junit4]   2> 934443 T4152 [] [] [] []  oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 934444 T4206 [collection1] [] [] [collection1]  oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 934444 T4206 [collection1] [] [] [collection1]  oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 934445 T4206 [collection1] [] [] [collection1]  oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 934445 T4178 [] [] [] []  oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 934446 T4179 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:61268_vbje%2Ft",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:61268/vbje/t",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1"} current state version: 4
   [junit4]   2> 934446 T4179 [] [] [] []  oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:61268_vbje%2Ft",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:61268/vbje/t",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 934447 T4179 [] [] [] []  oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 934447 T4179 [] [] [] []  oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 934448 T4205 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 934448 T4190 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 934448 T4178 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 935445 T4206 [collection1] [] [] [collection1]  oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 935446 T4206 [collection1] [] [] [collection1]  oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 935446 T4206 [collection1] [] [] [collection1]  oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 935447 T4206 [collection1] [] [] [collection1]  oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 935447 T4206 [collection1] [] [] [collection1]  oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 935447 T4206 [collection1] [] [] [collection1]  oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/shard-1-001/cores/collection1/'
   [junit4]   2> 935460 T4206 [collection1] [] [] [collection1]  oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 935462 T4206 [collection1] [] [] [collection1]  oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 935469 T4206 [collection1] [] [] [collection1]  oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 935485 T4206 [collection1] [] [] [collection1]  oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 935520 T4206 [collection1] [] [] [collection1]  oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 935621 T4206 [collection1] [] [] [collection1]  oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 935628 T4206 [collection1] [] [] [collection1]  oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 935906 T4206 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 935922 T4206 [collection1] [] [] [collection1]  oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 935925 T4206 [collection1] [] [] [collection1]  oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 935934 T4206 [collection1] [] [] [collection1]  oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 935939 T4206 [collection1] [] [] [collection1]  oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 935942 T4206 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 935943 T4206 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 935943 T4206 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 935944 T4206 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 935944 T4206 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 935945 T4206 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 935945 T4206 [collection1] [] [] [collection1]  oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 935945 T4206 [collection1] [] [] [collection1]  oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 935946 T4206 [collection1] [] [] [collection1]  oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 935946 T4206 [collection1] [] [] [collection1]  oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1f0fee7c
   [junit4]   2> 935947 T4206 [collection1] [] [] [collection1]  oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/shard-1-001/cores/collection1/data
   [junit4]   2> 935947 T4206 [collection1] [] [] [collection1]  oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 935947 T4206 [collection1] [] [] [collection1]  oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 935948 T4206 [collection1] [] [] [collection1]  oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest F890540B8D1E93B0-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 935948 T4206 [collection1] [] [] [collection1]  oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=17, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=18.5205078125, floorSegmentMB=0.8837890625, forceMergeDeletesPctAllowed=27.25970516083252, segmentsPerTier=10.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 935949 T4206 [collection1] [] [] [collection1]  oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1efcbdb3 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@630a42a9),segFN=segments_1,generation=1}
   [junit4]   2> 935950 T4206 [collection1] [] [] [collection1]  oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 935957 T4206 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 935957 T4206 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 935958 T4206 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 935958 T4206 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 935958 T4206 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 935959 T4206 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 935959 T4206 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 935960 T4206 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 935960 T4206 [collection1] [] [] [collection1]  oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 935978 T4206 [collection1] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 935981 T4206 [collection1] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 935983 T4206 [collection1] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 935985 T4206 [collection1] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 935994 T4206 [collection1] [] [] [collection1]  oasc.RequestHandlers.initHandlersFromConfig Registered paths: /get,/schema,/update/json/docs,/admin/luke,/replication,/admin/file,/admin/segments,/admin/plugins,/config,/admin/ping,/admin/threads,/update/json,/admin/logging,/admin/mbeans,/admin/system,standard,/update/csv,/update,/admin/properties
   [junit4]   2> 935996 T4206 [collection1] [] [] [collection1]  oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 935998 T4206 [collection1] [] [] [collection1]  oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 935998 T4206 [collection1] [] [] [collection1]  oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 935999 T4206 [collection1] [] [] [collection1]  oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 936000 T4206 [collection1] [] [] [collection1]  oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 936001 T4206 [collection1] [] [] [collection1]  oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=42, maxMergeAtOnceExplicit=33, maxMergedSegmentMB=79.6669921875, floorSegmentMB=1.30078125, forceMergeDeletesPctAllowed=11.071207434441273, segmentsPerTier=27.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.47216918996256496
   [junit4]   2> 936002 T4206 [collection1] [] [] [collection1]  oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1efcbdb3 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@630a42a9),segFN=segments_1,generation=1}
   [junit4]   2> 936002 T4206 [collection1] [] [] [collection1]  oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 936002 T4206 [collection1] [] [] [collection1]  oass.SolrIndexSearcher.<init> Opening Searcher@5725a757[collection1] main
   [junit4]   2> 936003 T4206 [collection1] [] [] [collection1]  oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 936003 T4206 [collection1] [] [] [collection1]  oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 936004 T4206 [collection1] [] [] [collection1]  oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 936004 T4206 [collection1] [] [] [collection1]  oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 936005 T4206 [collection1] [] [] [collection1]  oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 936005 T4206 [collection1] [] [] [collection1]  oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 936005 T4206 [collection1] [] [] [collection1]  oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 936006 T4206 [collection1] [] [] [collection1]  oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 936006 T4206 [collection1] [] [] [collection1]  oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 936006 T4206 [collection1] [] [] [collection1]  oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 936007 T4207 [collection1] [] [] [collection1]  oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5725a757[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 936008 T4206 [collection1] [] [] [collection1]  oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 936009 T4210 [collection1] [shard1] [] []  oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:61268/vbje/t collection:collection1 shard:shard1
   [junit4]   2> 936009 T4152 [] [] [] []  oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 936009 T4152 [] [] [] []  oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 936010 T4210 [collection1] [shard1] [] []  oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 936013 T4210 [collection1] [shard1] [] []  oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 936015 T4178 [] [] [] []  oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 936015 T4210 [collection1] [shard1] [] []  oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 936015 T4210 [collection1] [shard1] [] []  oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C2350 name=collection1 org.apache.solr.core.SolrCore@1096f782 url=http://127.0.0.1:61268/vbje/t/collection1 node=127.0.0.1:61268_vbje%2Ft C2350_STATE=coll:collection1 core:collection1 props:{node_name=127.0.0.1:61268_vbje%2Ft, base_url=http://127.0.0.1:61268/vbje/t, state=down, core=collection1}
   [junit4]   2> 936016 T4210 [collection1] [shard1] [] []  C2350 P61268 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:61268/vbje/t/collection1/
   [junit4]   2> 936016 T4179 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 936016 T4210 [collection1] [shard1] [] []  C2350 P61268 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 936016 T4210 [collection1] [shard1] [] []  C2350 P61268 oasc.SyncStrategy.syncToMe http://127.0.0.1:61268/vbje/t/collection1/ has no replicas
   [junit4]   2> 936017 T4210 [collection1] [shard1] [] []  oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:61268/vbje/t/collection1/ shard1
   [junit4]   2> 936017 T4210 [collection1] [shard1] [] []  oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 936017 T4178 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 936017 T4190 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 936017 T4205 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 936021 T4178 [] [] [] []  oasc.DistributedQueue$Lat

[...truncated too long message...]

ontrol_collection] [shard1] [] [collection1]  oasc.SocketProxy.close WARN Closing 10 connections to: http://127.0.0.1:48436/yd
   [junit4]   2> 1610877 T6795 [control_collection] [shard1] [] [collection1]  oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:48440/yd
   [junit4]   2> 1610878 T6795 [control_collection] [shard1] [] [collection1]  oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:48414/yd
   [junit4]   2> 1610878 T6795 [control_collection] [shard1] [] [collection1]  oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:48450/yd
   [junit4]   2> 1610878 T6795 [control_collection] [shard1] [] [collection1]  oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 1610878 T6795 [control_collection] [shard1] [] [collection1]  oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:48436/yd
   [junit4]   2> 1610879 T6795 [control_collection] [shard1] [] [collection1]  oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:48440/yd
   [junit4]   2> 1610879 T6795 [control_collection] [shard1] [] [collection1]  oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:48414/yd
   [junit4]   2> 1610879 T6795 [control_collection] [shard1] [] [collection1]  oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:48450/yd
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=F890540B8D1E93B0 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=es -Dtests.timezone=Pacific/Rarotonga -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   79.0s J2 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([F890540B8D1E93B0:70C46BD123E2FE48]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 1610901 T6795 [control_collection] [shard1] [] [collection1]  oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest F890540B8D1E93B0-001
   [junit4]   2> 78958 T6794 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=FastCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST, chunkSize=5100, maxDocsPerChunk=4, blockSize=10), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST, chunkSize=5100, blockSize=10)), sim=DefaultSimilarity, locale=es, timezone=Pacific/Rarotonga
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=198128696,total=421527552
   [junit4]   2> NOTE: All tests run in this JVM: [TestFunctionQuery, TestReloadAndDeleteDocs, NoCacheHeaderTest, TestWordDelimiterFilterFactory, AddSchemaFieldsUpdateProcessorFactoryTest, HardAutoCommitTest, AutoCommitTest, TestPartialUpdateDeduplication, TestCloudManagedSchemaConcurrent, TimeZoneUtilsTest, MoreLikeThisHandlerTest, PluginInfoTest, ShardRoutingCustomTest, TestNonDefinedSimilarityFactory, NotRequiredUniqueKeyTest, TestDefaultStatsCache, TestSortingResponseWriter, ResourceLoaderTest, RegexBoostProcessorTest, TestExceedMaxTermLength, TestDFRSimilarityFactory, SpellCheckComponentTest, OverriddenZkACLAndCredentialsProvidersTest, TestSystemIdResolver, TestDistributedSearch, TestStandardQParsers, DocumentAnalysisRequestHandlerTest, TestCopyFieldCollectionResource, HdfsWriteToMultipleCollectionsTest, TestHashPartitioner, FastVectorHighlighterTest, TestSearchPerf, TestRemoteStreaming, AsyncMigrateRouteKeyTest, SuggesterTest, TestCollectionAPI, TestReplicaProperties, TestLuceneMatchVersion, DistributedSuggestComponentTest, BlockDirectoryTest, CollectionsAPIAsyncDistributedZkTest, BadIndexSchemaTest, SampleTest, TestClassNameShortening, LoggingHandlerTest, TestSort, TestReplicationHandlerBackup, TestRandomMergePolicy, RecoveryAfterSoftCommitTest, SoftAutoCommitTest, TestHighlightDedupGrouping, DistributedFacetPivotLongTailTest, TestSolrQueryParser, TestShardHandlerFactory, SpellingQueryConverterTest, TestReplicationHandler, CollectionsAPIDistributedZkTest, BasicDistributedZkTest, CustomCollectionTest, ReplicationFactorTest]
   [junit4] Completed on J2 in 79.75s, 1 test, 1 error <<< FAILURES!

[...truncated 722 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 476 suites, 1896 tests, 3 errors, 62 ignored (22 assumptions)

Total time: 54 minutes 36 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 221164337 bytes
Compression is 0.0%
Took 43 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2832 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2832/

4 tests failed.
REGRESSION:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test

Error Message:
There were too many update fails - we expect it can happen, but shouldn't easily

Stack Trace:
java.lang.AssertionError: There were too many update fails - we expect it can happen, but shouldn't easily
	at __randomizedtesting.SeedInfo.seed([DE227811035DE907:567647CBADA184FF]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertFalse(Assert.java:68)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:222)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:49334/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:49334/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([DE227811035DE907:567647CBADA184FF]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:18840/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:18840/collection1
	at __randomizedtesting.SeedInfo.seed([DE227811035DE907:567647CBADA184FF]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:18840/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:14738/repfacttest_c8n_1x3_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:14738/repfacttest_c8n_1x3_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([DE227811035DE907:567647CBADA184FF]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:14738/repfacttest_c8n_1x3_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 53 more




Build Log:
[...truncated 9363 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/init-core-data-001
   [junit4]   2> 238665 T645 [] [] [] []  oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 238677 T645 [] [] [] []  oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 238679 T646 [] [] [] []  oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 238778 T645 [] [] [] []  oasc.ZkTestServer.run start zk server on port:49302
   [junit4]   2> 238789 T645 [] [] [] []  oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 238792 T645 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 238802 T653 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@585724f name:ZooKeeperConnection Watcher:127.0.0.1:49302 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 238803 T645 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 238804 T645 [] [] [] []  oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 238805 T645 [] [] [] []  oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 238812 T645 [] [] [] []  oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 238814 T645 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 238817 T656 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@63cde8e name:ZooKeeperConnection Watcher:127.0.0.1:49302/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 238818 T645 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 238819 T645 [] [] [] []  oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 238820 T645 [] [] [] []  oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 238828 T645 [] [] [] []  oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 238835 T645 [] [] [] []  oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 238840 T645 [] [] [] []  oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 238846 T645 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 238847 T645 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 238856 T645 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 238857 T645 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 238962 T645 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 238964 T645 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 238969 T645 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 238970 T645 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 238978 T645 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 238980 T645 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 238988 T645 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 238991 T645 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 238999 T645 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 239000 T645 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 239006 T645 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 239008 T645 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 239013 T645 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 239014 T645 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 239019 T645 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 239021 T645 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 239026 T645 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 239027 T645 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 241900 T645 [] [] [] []  oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/control-001/cores/collection1
   [junit4]   2> 241911 T645 [] [] [] []  oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 241918 T645 [] [] [] []  oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:49309
   [junit4]   2> 241919 T645 [] [] [] []  oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/tempDir-001/control/data, hostContext=/, hostPort=49308, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/control-001/cores}
   [junit4]   2> 241920 T645 [] [] [] []  oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 241921 T645 [] [] [] []  oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/control-001/'
   [junit4]   2> 242036 T645 [] [] [] []  oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/control-001/solr.xml
   [junit4]   2> 242174 T645 [] [] [] []  oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/control-001/cores
   [junit4]   2> 242175 T645 [] [] [] []  oasc.CoreContainer.<init> New CoreContainer 31159509
   [junit4]   2> 242176 T645 [] [] [] []  oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/control-001/]
   [junit4]   2> 242176 T645 [] [] [] []  oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/control-001/lib
   [junit4]   2> 242179 T645 [] [] [] []  oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/control-001/lib).
   [junit4]   2> 242212 T645 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 242213 T645 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 242214 T645 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 242214 T645 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 242215 T645 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 242216 T645 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 242216 T645 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 242217 T645 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 242218 T645 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 242218 T645 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 242219 T645 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 242220 T645 [] [] [] []  oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 242221 T645 [] [] [] []  oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 242222 T645 [] [] [] []  oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 242222 T645 [] [] [] []  oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 242223 T645 [] [] [] []  oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49302/solr
   [junit4]   2> 242223 T645 [] [] [] []  oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 242224 T645 [] [] [] []  oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 242226 T645 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 242238 T668 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3642bccd name:ZooKeeperConnection Watcher:127.0.0.1:49302 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 242239 T645 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 242240 T645 [] [] [] []  oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 242245 T645 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 242248 T671 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@62752dd0 name:ZooKeeperConnection Watcher:127.0.0.1:49302/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 242249 T645 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 242255 T645 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 242264 T645 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 242273 T645 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 242281 T645 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 242291 T645 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 242298 T645 [] [] [] []  oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 242303 T645 [] [] [] []  oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 242307 T645 [] [] [] []  oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 242310 T645 [] [] [] []  oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49308_
   [junit4]   2> 242312 T645 [] [] [] []  oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49308_
   [junit4]   2> 242316 T645 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 242319 T645 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 242325 T645 [] [] [] []  oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 242330 T645 [] [] [] []  oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:49308_
   [junit4]   2> 242331 T645 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 242336 T645 [] [] [] []  oasc.Overseer.start Overseer (id=93539994084245507-127.0.0.1:49308_-n_0000000000) starting
   [junit4]   2> 242343 T645 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 242424 T645 [] [] [] []  oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 242425 T673 [] [] [] []  oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 242425 T645 [] [] [] []  oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 242428 T672 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 242453 T645 [] [] [] []  oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/control-001/cores
   [junit4]   2> 242472 T645 [] [] [] []  oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {schema=schema.xml, collection=control_collection, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/control-001/cores/collection1/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/control-001/cores/collection1, name=collection1, coreNodeName=, dataDir=data/, transient=false, loadOnStartup=true, config=solrconfig.xml, shard=}
   [junit4]   2> 242473 T645 [] [] [] []  oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/control-001/cores/collection1/
   [junit4]   2> 242474 T645 [] [] [] []  oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 242500 T675 [control_collection] [] [] [collection1]  oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 242501 T675 [control_collection] [] [] [collection1]  oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 242524 T671 [] [] [] []  oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 242525 T675 [control_collection] [] [] [collection1]  oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 242528 T672 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:49308_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49308",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down"} current state version: 0
   [junit4]   2> 242660 T672 [] [] [] []  oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:49308_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49308",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 242661 T672 [] [] [] []  oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 242747 T672 [] [] [] []  oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 242795 T671 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 243527 T675 [control_collection] [] [] [collection1]  oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 243527 T675 [control_collection] [] [] [collection1]  oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 243529 T675 [control_collection] [] [] [collection1]  oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 243529 T675 [control_collection] [] [] [collection1]  oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 243531 T675 [control_collection] [] [] [collection1]  oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 243532 T675 [control_collection] [] [] [collection1]  oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/control-001/cores/collection1/'
   [junit4]   2> 243581 T675 [control_collection] [] [] [collection1]  oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 243607 T675 [control_collection] [] [] [collection1]  oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 243642 T675 [control_collection] [] [] [collection1]  oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 243671 T675 [control_collection] [] [] [collection1]  oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 243729 T675 [control_collection] [] [] [collection1]  oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 243832 T675 [control_collection] [] [] [collection1]  oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 243844 T675 [control_collection] [] [] [collection1]  oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 244497 T675 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 244519 T675 [control_collection] [] [] [collection1]  oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 244524 T675 [control_collection] [] [] [collection1]  oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 244563 T675 [control_collection] [] [] [collection1]  oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 244572 T675 [control_collection] [] [] [collection1]  oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 244579 T675 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 244582 T675 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 244583 T675 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 244583 T675 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 244587 T675 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 244587 T675 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 244588 T675 [control_collection] [] [] [collection1]  oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 244588 T675 [control_collection] [] [] [collection1]  oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 244589 T675 [control_collection] [] [] [collection1]  oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/control-001/cores/collection1/data/
   [junit4]   2> 244590 T675 [control_collection] [] [] [collection1]  oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@572329c9
   [junit4]   2> 244591 T675 [control_collection] [] [] [collection1]  oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/control-001/cores/collection1/data
   [junit4]   2> 244592 T675 [control_collection] [] [] [collection1]  oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/control-001/cores/collection1/data/index/
   [junit4]   2> 244592 T675 [control_collection] [] [] [collection1]  oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 244593 T675 [control_collection] [] [] [collection1]  oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/control-001/cores/collection1/data/index
   [junit4]   2> 244593 T675 [control_collection] [] [] [collection1]  oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=30, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 244594 T675 [control_collection] [] [] [collection1]  oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2e230c34 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7ef64195),segFN=segments_1,generation=1}
   [junit4]   2> 244595 T675 [control_collection] [] [] [collection1]  oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 244602 T675 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 244603 T675 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 244603 T675 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 244603 T675 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 244604 T675 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 244604 T675 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 244605 T675 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 244605 T675 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 244606 T675 [control_collection] [] [] [collection1]  oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 244639 T675 [control_collection] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 244645 T675 [control_collection] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 244650 T675 [control_collection] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 244654 T675 [control_collection] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 244664 T675 [control_collection] [] [] [collection1]  oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/file,/admin/system,/get,/admin/ping,/config,/replication,/update/csv,/admin/mbeans,standard,/admin/segments,/admin/plugins,/admin/luke,/update,/admin/properties,/admin/logging,/update/json/docs,/update/json,/schema,/admin/threads
   [junit4]   2> 244668 T675 [control_collection] [] [] [collection1]  oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 244672 T675 [control_collection] [] [] [collection1]  oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 244672 T675 [control_collection] [] [] [collection1]  oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 244675 T675 [control_collection] [] [] [collection1]  oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 244676 T675 [control_collection] [] [] [collection1]  oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 244677 T675 [control_collection] [] [] [collection1]  oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=14, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.37737859740630253]
   [junit4]   2> 244679 T675 [control_collection] [] [] [collection1]  oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2e230c34 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7ef64195),segFN=segments_1,generation=1}
   [junit4]   2> 244679 T675 [control_collection] [] [] [collection1]  oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 244680 T675 [control_collection] [] [] [collection1]  oass.SolrIndexSearcher.<init> Opening Searcher@72a387e9[collection1] main
   [junit4]   2> 244681 T675 [control_collection] [] [] [collection1]  oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 244683 T675 [control_collection] [] [] [collection1]  oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 244684 T675 [control_collection] [] [] [collection1]  oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 244709 T675 [control_collection] [] [] [collection1]  oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 244709 T675 [control_collection] [] [] [collection1]  oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 244710 T675 [control_collection] [] [] [collection1]  oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 244712 T675 [control_collection] [] [] [collection1]  oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 244712 T675 [control_collection] [] [] [collection1]  oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 244713 T675 [control_collection] [] [] [collection1]  oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 244713 T675 [control_collection] [] [] [collection1]  oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 244715 T676 [control_collection] [] [] [collection1]  oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@72a387e9[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 244732 T675 [control_collection] [] [] [collection1]  oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 244792 T679 [control_collection] [shard1] [] []  oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:49308 collection:control_collection shard:shard1
   [junit4]   2> 244793 T645 [] [] [] []  oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 244793 T645 [] [] [] []  oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 244810 T645 [] [] [] []  oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 244811 T645 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 244814 T683 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@66e56fc7 name:ZooKeeperConnection Watcher:127.0.0.1:49302/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 244815 T645 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 244816 T645 [] [] [] []  oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 244817 T645 [] [] [] []  oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 244831 T645 [] [] [] []  oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 244860 T679 [control_collection] [shard1] [] []  oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 244880 T679 [control_collection] [shard1] [] []  oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 244885 T671 [] [] [] []  oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 244887 T679 [control_collection] [shard1] [] []  oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 244887 T679 [control_collection] [shard1] [] []  oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C55 name=collection1 org.apache.solr.core.SolrCore@553ae232 url=http://127.0.0.1:49308/collection1 node=127.0.0.1:49308_ C55_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:49308_, base_url=http://127.0.0.1:49308, core=collection1, state=down}
   [junit4]   2> 244888 T679 [control_collection] [shard1] [] []  C55 P49308 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:49308/collection1/
   [junit4]   2> 244888 T672 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 244889 T679 [control_collection] [shard1] [] []  C55 P49308 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 244897 T679 [control_collection] [shard1] [] []  C55 P49308 oasc.SyncStrategy.syncToMe http://127.0.0.1:49308/collection1/ has no replicas
   [junit4]   2> 244897 T679 [control_collection] [shard1] [] []  oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:49308/collection1/ shard1
   [junit4]   2> 244905 T683 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 244906 T671 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 244975 T679 [control_collection] [shard1] [] []  oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 245014 T671 [] [] [] []  oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 245016 T672 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49308",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 245126 T683 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 245127 T671 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 245155 T679 [control_collection] [shard1] [] []  oasc.ZkController.register We are http://127.0.0.1:49308/collection1/ and leader is http://127.0.0.1:49308/collection1/
   [junit4]   2> 245155 T679 [control_collection] [shard1] [] []  oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:49308
   [junit4]   2> 245156 T679 [control_collection] [shard1] [] []  oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 245156 T679 [control_collection] [shard1] [] [collection1]  oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 245156 T679 [control_collection] [shard1] [] [collection1]  oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 245202 T671 [] [] [] []  oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 245205 T672 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:49308_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49308",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 3
   [junit4]   2> 245207 T672 [] [] [] []  oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:49308_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49308",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"}
   [junit4]   2> 245319 T671 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 245319 T683 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 248026 T645 [] [] [] []  oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/shard-1-001/cores/collection1
   [junit4]   2> 248030 T645 [] [] [] []  oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/shard-1-001
   [junit4]   2> 248031 T645 [] [] [] []  oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 248035 T645 [] [] [] []  oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:49319
   [junit4]   2> 248036 T645 [] [] [] []  oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/tempDir-001/jetty1, hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/shard-1-001/cores, solrconfig=solrconfig.xml, hostPort=49318}
   [junit4]   2> 248036 T645 [] [] [] []  oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 248037 T645 [] [] [] []  oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/shard-1-001/'
   [junit4]   2> 248114 T645 [] [] [] []  oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/shard-1-001/solr.xml
   [junit4]   2> 248198 T645 [] [] [] []  oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/shard-1-001/cores
   [junit4]   2> 248199 T645 [] [] [] []  oasc.CoreContainer.<init> New CoreContainer 1495965591
   [junit4]   2> 248200 T645 [] [] [] []  oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/shard-1-001/]
   [junit4]   2> 248201 T645 [] [] [] []  oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/shard-1-001/lib
   [junit4]   2> 248202 T645 [] [] [] []  oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/shard-1-001/lib).
   [junit4]   2> 248230 T645 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 248231 T645 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 248232 T645 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 248232 T645 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 248233 T645 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 248234 T645 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 248235 T645 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 248235 T645 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 248236 T645 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 248237 T645 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 248237 T645 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 248239 T645 [] [] [] []  oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 248240 T645 [] [] [] []  oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 248241 T645 [] [] [] []  oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 248241 T645 [] [] [] []  oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 248242 T645 [] [] [] []  oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49302/solr
   [junit4]   2> 248243 T645 [] [] [] []  oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 248244 T645 [] [] [] []  oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 248250 T645 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 248254 T695 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6bdbd792 name:ZooKeeperConnection Watcher:127.0.0.1:49302 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 248256 T645 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 248258 T645 [] [] [] []  oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 248290 T645 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 248292 T698 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@71628ca1 name:ZooKeeperConnection Watcher:127.0.0.1:49302/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 248293 T645 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 248312 T645 [] [] [] []  oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 249389 T645 [] [] [] []  oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49318_
   [junit4]   2> 249391 T645 [] [] [] []  oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49318_
   [junit4]   2> 249400 T645 [] [] [] []  oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 249407 T645 [] [] [] []  oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/shard-1-001/cores
   [junit4]   2> 249413 T645 [] [] [] []  oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/shard-1-001/cores/collection1, name=collection1, transient=false, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/shard-1-001/cores/collection1/, config=solrconfig.xml, dataDir=data/, coreNodeName=, loadOnStartup=true, schema=schema.xml, shard=, collection=collection1}
   [junit4]   2> 249414 T645 [] [] [] []  oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/shard-1-001/cores/collection1/
   [junit4]   2> 249415 T645 [] [] [] []  oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 249418 T699 [collection1] [] [] [collection1]  oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 249419 T699 [collection1] [] [] [collection1]  oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 249421 T671 [] [] [] []  oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 249421 T699 [collection1] [] [] [collection1]  oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 249424 T672 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:49318_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49318",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down"} current state version: 4
   [junit4]   2> 249426 T672 [] [] [] []  oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:49318_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49318",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 249426 T672 [] [] [] []  oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 249427 T672 [] [] [] []  oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 249431 T683 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 249431 T698 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 249431 T671 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 250423 T699 [collection1] [] [] [collection1]  oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 250423 T699 [collection1] [] [] [collection1]  oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 250425 T699 [collection1] [] [] [collection1]  oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 250425 T699 [collection1] [] [] [collection1]  oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 250427 T699 [collection1] [] [] [collection1]  oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 250427 T699 [collection1] [] [] [collection1]  oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/shard-1-001/cores/collection1/'
   [junit4]   2> 250453 T699 [collection1] [] [] [collection1]  oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 250458 T699 [collection1] [] [] [collection1]  oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 250475 T699 [collection1] [] [] [collection1]  oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 250499 T699 [collection1] [] [] [collection1]  oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 250558 T699 [collection1] [] [] [collection1]  oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 250661 T699 [collection1] [] [] [collection1]  oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 250674 T699 [collection1] [] [] [collection1]  oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 251338 T699 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 251373 T699 [collection1] [] [] [collection1]  oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 251378 T699 [collection1] [] [] [collection1]  oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 251409 T699 [collection1] [] [] [collection1]  oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 251418 T699 [collection1] [] [] [collection1]  oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 251425 T699 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 251428 T699 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 251428 T699 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 251429 T699 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 251431 T699 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 251431 T699 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 251432 T699 [collection1] [] [] [collection1]  oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 251432 T699 [collection1] [] [] [collection1]  oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 251433 T699 [collection1] [] [] [collection1]  oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 251433 T699 [collection1] [] [] [collection1]  oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@572329c9
   [junit4]   2> 251434 T699 [collection1] [] [] [collection1]  oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/shard-1-001/cores/collection1/data
   [junit4]   2> 251435 T699 [collection1] [] [] [collection1]  oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 251435 T699 [collection1] [] [] [collection1]  oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 251435 T699 [collection1] [] [] [collection1]  oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DE227811035DE907-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 251436 T699 [collection1] [] [] [collection1]  oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=30, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 251437 T699 [collection1] [] [] [collection1]  oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1f491ded lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@794bc2a6),segFN=segments_1,generation=1}
   [junit4]   2> 251437 T699 [collection1] [] [] [collection1]  oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 251451 T699 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 251452 T699 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 251453 T699 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 251453 T699 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 251454 T699 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 251454 T699 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 251455 T699 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 251456 T699 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 251457 T699 [collection1] [] [] [collection1]  oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 251500 T699 [collection1] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 251507 T699 [collection1] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 251513 T699 [collection1] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 251519 T699 [collection1] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 251536 T699 [collection1] [] [] [collection1]  oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/file,/admin/system,/get,/admin/ping,/config,/replication,/update/csv,/admin/mbeans,standard,/admin/segments,/admin/plugins,/admin/luke,/update,/admin/properties,/admin/logging,/update/json/docs,/update/json,/schema,/admin/threads
   [junit4]   2> 251540 T699 [collection1] [] [] [collection1]  oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 251544 T699 [collection1] [] [] [collection1]  oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 251544 T699 [collection1] [] [] [collection1]  oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 251548 T699 [collection1] [] [] [collection1]  oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 251548 T699 [collection1] [] [] [collection1]  oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 251550 T699 [collection1] [] [] [collection1]  oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=14, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.37737859740630253]
   [junit4]   2> 251552 T699 [collection1] [] [] [collection1]  oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1f491ded lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@794bc2a6),segFN=segments_1,generation=1}
   [junit4]   2> 251552 T699 [collection1] [] [] [collection1]  oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 251553 T699 [collection1] [] [] [collection1]  oass.SolrIndexSearcher.<init> Opening Searcher@1b263d21[collection1] main
   [junit4]   2> 251554 T699 [collection1] [] [] [collection1]  oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 251557 T699 [collection1] [] [] [collection1]  oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 251558 T699 [collection1] [] [] [collection1]  oasr.ManagedResourceStorage

[...truncated too long message...]

junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1547240 T5117 [control_collection] [shard1] [] [collection1]  oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:18847/
   [junit4]   2> 1547240 T5117 [control_collection] [shard1] [] [collection1]  oasc.SocketProxy.close WARN Closing 6 connections to: http://127.0.0.1:18840/
   [junit4]   2> 1547241 T5117 [control_collection] [shard1] [] [collection1]  oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:18834/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=DE227811035DE907 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=et_EE -Dtests.timezone=America/Argentina/San_Juan -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   72.2s J3 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:18840/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([DE227811035DE907:567647CBADA184FF]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:18840/collection1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 49 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
   [junit4]    > 	... 54 more
   [junit4]   2> 1547276 T5117 [control_collection] [shard1] [] [collection1]  oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest DE227811035DE907-001
   [junit4]   2> 72161 T5116 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {_version_=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), text=PostingsFormat(name=MockRandom), intDefault=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), multiDefault=PostingsFormat(name=LuceneFixedGap), timestamp=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), id=Lucene50(blocksize=128), range_facet_l=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), a_t=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128)))}, docValues:{timestamp=DocValuesFormat(name=Memory)}, sim=RandomSimilarityProvider(queryNorm=false,coord=yes): {}, locale=et_EE, timezone=America/Argentina/San_Juan
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=133986704,total=468713472
   [junit4]   2> NOTE: All tests run in this JVM: [TestCloudManagedSchemaConcurrent, TestSurroundQueryParser, BasicDistributedZk2Test, AnalyticsQueryTest, RAMDirectoryFactoryTest, TestManagedSchemaFieldTypeResource, TestIntervalFaceting, DistributedQueryElevationComponentTest, TestInitQParser, TestBinaryResponseWriter, TestArbitraryIndexDir, TestJmxIntegration, PreAnalyzedUpdateProcessorTest, TestSuggestSpellingConverter, TestFastLRUCache, TestCopyFieldCollectionResource, TestSolr4Spatial, TestBulkSchemaAPI, PluginInfoTest, TestSort, TestRealTimeGet, DirectSolrConnectionTest, ZkStateWriterTest, OverriddenZkACLAndCredentialsProvidersTest, TestManagedSchemaDynamicFieldResource, BlockDirectoryTest, DisMaxRequestHandlerTest, TestJsonFacets, SOLR749Test, QueryResultKeyTest, TestFunctionQuery, TestCoreDiscovery, HdfsUnloadDistributedZkTest, TestQueryUtils, TestSortingResponseWriter, DirectUpdateHandlerOptimizeTest, TestSolrQueryParserDefaultOperatorResource, XsltUpdateRequestHandlerTest, TestSolrDeletionPolicy2, TestBM25SimilarityFactory, TestConfigOverlay, TestLMDirichletSimilarityFactory, DeleteShardTest, SuggesterWFSTTest, QueryParsingTest, NoCacheHeaderTest, TestObjectReleaseTracker, TimeZoneUtilsTest, TestComplexPhraseQParserPlugin, TestAnalyzedSuggestions, TestFreeTextSuggestions, SharedFSAutoReplicaFailoverTest, TestDocSet, CloudExitableDirectoryReaderTest, TestPerFieldSimilarity, StatsComponentTest, TestDistributedSearch, TestCloudSchemaless, CustomCollectionTest, TestSolrConfigHandlerCloud, DistribCursorPagingTest, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J3 in 72.69s, 1 test, 1 error <<< FAILURES!

[...truncated 693 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 475 suites, 1894 tests, 3 errors, 1 failure, 62 ignored (22 assumptions)

Total time: 54 minutes 37 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 231946877 bytes
Compression is 0.0%
Took 42 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2831 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2831/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:18943/ak/zh/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:18943/ak/zh/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([212FDD5EC8B0F2CF:A97BE284664C9F37]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([212FDD5EC8B0F2CF:A97BE284664C9F37]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([212FDD5EC8B0F2CF:A97BE284664C9F37]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9248 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/init-core-data-001
   [junit4]   2> 100583 T279 [] [] [] []  oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 100595 T279 [] [] [] []  oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 100598 T280 [] [] [] []  oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 100698 T279 [] [] [] []  oasc.ZkTestServer.run start zk server on port:34883
   [junit4]   2> 100699 T279 [] [] [] []  oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 100701 T279 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 100707 T287 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5e4680df name:ZooKeeperConnection Watcher:127.0.0.1:34883 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 100707 T279 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 100708 T279 [] [] [] []  oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 100709 T279 [] [] [] []  oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 100714 T279 [] [] [] []  oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 100716 T279 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 100718 T290 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@328b7fd3 name:ZooKeeperConnection Watcher:127.0.0.1:34883/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 100719 T279 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 100720 T279 [] [] [] []  oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 100720 T279 [] [] [] []  oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 100726 T279 [] [] [] []  oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 100730 T279 [] [] [] []  oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 100734 T279 [] [] [] []  oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 100740 T279 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 100741 T279 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 100748 T279 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 100749 T279 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 100755 T279 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 100756 T279 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 100762 T279 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 100763 T279 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 100768 T279 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 100769 T279 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 100774 T279 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 100775 T279 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 100780 T279 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 100781 T279 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 100786 T279 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 100787 T279 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 100793 T279 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 100793 T279 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 100798 T279 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 100799 T279 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 100805 T279 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 100806 T279 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 102094 T279 [] [] [] []  oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/control-001/cores/collection1
   [junit4]   2> 102098 T279 [] [] [] []  oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 102101 T279 [] [] [] []  oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:34890
   [junit4]   2> 102101 T279 [] [] [] []  oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/control-001/cores, hostContext=/, hostPort=34889, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/tempDir-001/control/data}
   [junit4]   2> 102102 T279 [] [] [] []  oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3972aa3f
   [junit4]   2> 102102 T279 [] [] [] []  oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/control-001/'
   [junit4]   2> 102134 T279 [] [] [] []  oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/control-001/solr.xml
   [junit4]   2> 102172 T279 [] [] [] []  oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/control-001/cores
   [junit4]   2> 102173 T279 [] [] [] []  oasc.CoreContainer.<init> New CoreContainer 1597661195
   [junit4]   2> 102173 T279 [] [] [] []  oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/control-001/]
   [junit4]   2> 102173 T279 [] [] [] []  oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/control-001/lib
   [junit4]   2> 102174 T279 [] [] [] []  oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/control-001/lib).
   [junit4]   2> 102183 T279 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 102184 T279 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 102184 T279 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 102184 T279 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 102185 T279 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 102185 T279 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 102185 T279 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 102185 T279 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 102186 T279 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 102186 T279 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 102186 T279 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 102187 T279 [] [] [] []  oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 102187 T279 [] [] [] []  oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 102188 T279 [] [] [] []  oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 102188 T279 [] [] [] []  oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 102188 T279 [] [] [] []  oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:34883/solr
   [junit4]   2> 102189 T279 [] [] [] []  oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 102189 T279 [] [] [] []  oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 102190 T279 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 102192 T302 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@433a32ea name:ZooKeeperConnection Watcher:127.0.0.1:34883 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 102193 T279 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 102193 T279 [] [] [] []  oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 102196 T279 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 102198 T305 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6a7b21cb name:ZooKeeperConnection Watcher:127.0.0.1:34883/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 102198 T279 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 102201 T279 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 102204 T279 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 102208 T279 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 102211 T279 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 102215 T279 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 102219 T279 [] [] [] []  oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 102222 T279 [] [] [] []  oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 102225 T279 [] [] [] []  oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 102227 T279 [] [] [] []  oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34889_
   [junit4]   2> 102228 T279 [] [] [] []  oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:34889_
   [junit4]   2> 102231 T279 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 102234 T279 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 102236 T279 [] [] [] []  oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 102239 T279 [] [] [] []  oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:34889_
   [junit4]   2> 102239 T279 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 102242 T279 [] [] [] []  oasc.Overseer.start Overseer (id=93538602098294787-127.0.0.1:34889_-n_0000000000) starting
   [junit4]   2> 102245 T279 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 102256 T279 [] [] [] []  oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 102257 T307 [] [] [] []  oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 102257 T279 [] [] [] []  oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 102258 T306 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 102264 T279 [] [] [] []  oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/control-001/cores
   [junit4]   2> 102275 T279 [] [] [] []  oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, transient=false, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/control-001/cores/collection1, name=collection1, collection=control_collection, dataDir=data/, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/control-001/cores/collection1/, shard=, coreNodeName=, loadOnStartup=true}
   [junit4]   2> 102275 T279 [] [] [] []  oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/control-001/cores/collection1/
   [junit4]   2> 102276 T279 [] [] [] []  oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 102277 T309 [control_collection] [] [] [collection1]  oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 102277 T309 [control_collection] [] [] [collection1]  oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 102284 T305 [] [] [] []  oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 102284 T309 [control_collection] [] [] [collection1]  oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 102285 T306 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:34889",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:34889_",
   [junit4]   2> 	  "shard":null} current state version: 0
   [junit4]   2> 102322 T306 [] [] [] []  oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:34889",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:34889_",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 102327 T306 [] [] [] []  oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 102358 T306 [] [] [] []  oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 102370 T305 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 103285 T309 [control_collection] [] [] [collection1]  oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 103285 T309 [control_collection] [] [] [collection1]  oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 103286 T309 [control_collection] [] [] [collection1]  oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 103286 T309 [control_collection] [] [] [collection1]  oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 103288 T309 [control_collection] [] [] [collection1]  oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 103288 T309 [control_collection] [] [] [collection1]  oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/control-001/cores/collection1/'
   [junit4]   2> 103320 T309 [control_collection] [] [] [collection1]  oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 103340 T309 [control_collection] [] [] [collection1]  oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 103362 T309 [control_collection] [] [] [collection1]  oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 103382 T309 [control_collection] [] [] [collection1]  oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 103420 T309 [control_collection] [] [] [collection1]  oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 103422 T309 [control_collection] [] [] [collection1]  oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 103430 T309 [control_collection] [] [] [collection1]  oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 103929 T309 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 103944 T309 [control_collection] [] [] [collection1]  oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 103947 T309 [control_collection] [] [] [collection1]  oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 103970 T309 [control_collection] [] [] [collection1]  oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 103974 T309 [control_collection] [] [] [collection1]  oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 103979 T309 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 103981 T309 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 103982 T309 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 103982 T309 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 103984 T309 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 103984 T309 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 103984 T309 [control_collection] [] [] [collection1]  oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 103985 T309 [control_collection] [] [] [collection1]  oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 103985 T309 [control_collection] [] [] [collection1]  oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/control-001/cores/collection1/data/
   [junit4]   2> 103986 T309 [control_collection] [] [] [collection1]  oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7ac28e11
   [junit4]   2> 103987 T309 [control_collection] [] [] [collection1]  oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/control-001/cores/collection1/data
   [junit4]   2> 103987 T309 [control_collection] [] [] [collection1]  oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/control-001/cores/collection1/data/index/
   [junit4]   2> 103987 T309 [control_collection] [] [] [collection1]  oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 103988 T309 [control_collection] [] [] [collection1]  oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/control-001/cores/collection1/data/index
   [junit4]   2> 103988 T309 [control_collection] [] [] [collection1]  oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=40, maxMergeAtOnceExplicit=41, maxMergedSegmentMB=77.2353515625, floorSegmentMB=0.7685546875, forceMergeDeletesPctAllowed=18.270899036203893, segmentsPerTier=21.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 103989 T309 [control_collection] [] [] [collection1]  oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@251fb3dd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@44598535),segFN=segments_1,generation=1}
   [junit4]   2> 103990 T309 [control_collection] [] [] [collection1]  oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 103997 T309 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 104069 T309 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 104070 T309 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 104070 T309 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 104070 T309 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 104071 T309 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 104072 T309 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 104072 T309 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 104072 T309 [control_collection] [] [] [collection1]  oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 104095 T309 [control_collection] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 104099 T309 [control_collection] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 104102 T309 [control_collection] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 104105 T309 [control_collection] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 104113 T309 [control_collection] [] [] [collection1]  oasc.RequestHandlers.initHandlersFromConfig Registered paths: standard,/update/csv,/admin/file,/admin/system,/admin/threads,/config,/update/json/docs,/admin/mbeans,/update/json,/admin/plugins,/admin/properties,/admin/segments,/schema,/admin/luke,/admin/ping,/update,/replication,/admin/logging,/get
   [junit4]   2> 104115 T309 [control_collection] [] [] [collection1]  oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 104117 T309 [control_collection] [] [] [collection1]  oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 104118 T309 [control_collection] [] [] [collection1]  oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 104120 T309 [control_collection] [] [] [collection1]  oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 104120 T309 [control_collection] [] [] [collection1]  oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 104121 T309 [control_collection] [] [] [collection1]  oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=10, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5845454432213795]
   [junit4]   2> 104122 T309 [control_collection] [] [] [collection1]  oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@251fb3dd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@44598535),segFN=segments_1,generation=1}
   [junit4]   2> 104123 T309 [control_collection] [] [] [collection1]  oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 104124 T309 [control_collection] [] [] [collection1]  oass.SolrIndexSearcher.<init> Opening Searcher@1e6acf9b[collection1] main
   [junit4]   2> 104125 T309 [control_collection] [] [] [collection1]  oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 104126 T309 [control_collection] [] [] [collection1]  oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 104127 T309 [control_collection] [] [] [collection1]  oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 104140 T309 [control_collection] [] [] [collection1]  oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 104140 T309 [control_collection] [] [] [collection1]  oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 104141 T309 [control_collection] [] [] [collection1]  oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 104142 T309 [control_collection] [] [] [collection1]  oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 104142 T309 [control_collection] [] [] [collection1]  oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 104142 T309 [control_collection] [] [] [collection1]  oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 104143 T309 [control_collection] [] [] [collection1]  oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 104144 T310 [control_collection] [] [] [collection1]  oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1e6acf9b[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 104151 T309 [control_collection] [] [] [collection1]  oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 104153 T313 [control_collection] [shard1] [] []  oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:34889 collection:control_collection shard:shard1
   [junit4]   2> 104154 T279 [] [] [] []  oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 104154 T279 [] [] [] []  oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 104183 T313 [control_collection] [shard1] [] []  oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 104190 T279 [] [] [] []  oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 104191 T279 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 104193 T317 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@f53a5d7 name:ZooKeeperConnection Watcher:127.0.0.1:34883/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 104193 T279 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 104194 T279 [] [] [] []  oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 104194 T313 [control_collection] [shard1] [] []  oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 104195 T279 [] [] [] []  oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 104198 T305 [] [] [] []  oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 104199 T313 [control_collection] [shard1] [] []  oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 104199 T313 [control_collection] [shard1] [] []  oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C25 name=collection1 org.apache.solr.core.SolrCore@1cbd6c5e url=http://127.0.0.1:34889/collection1 node=127.0.0.1:34889_ C25_STATE=coll:control_collection core:collection1 props:{core=collection1, base_url=http://127.0.0.1:34889, state=down, node_name=127.0.0.1:34889_}
   [junit4]   2> 104200 T313 [control_collection] [shard1] [] []  C25 P34889 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:34889/collection1/
   [junit4]   2> 104200 T306 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 104200 T313 [control_collection] [shard1] [] []  C25 P34889 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 104201 T313 [control_collection] [shard1] [] []  C25 P34889 oasc.SyncStrategy.syncToMe http://127.0.0.1:34889/collection1/ has no replicas
   [junit4]   2> 104201 T313 [control_collection] [shard1] [] []  oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:34889/collection1/ shard1
   [junit4]   2> 104205 T279 [] [] [] []  oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 104206 T279 [] [] [] []  oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 104206 T279 [] [] [] []  oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 104208 T279 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 104210 T320 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2269f993 name:ZooKeeperConnection Watcher:127.0.0.1:34883/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 104210 T279 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 104211 T279 [] [] [] []  oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 104213 T317 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 104213 T305 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 104216 T313 [control_collection] [shard1] [] []  oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 104224 T306 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 104224 T306 [] [] [] []  oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 104229 T305 [] [] [] []  oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 104232 T306 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:34889",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 104234 T306 [] [] [] []  oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 104236 T317 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 104236 T305 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 104245 T321 [] [] [] []  oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 104246 T321 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 104246 T322 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 104282 T313 [control_collection] [shard1] [] []  oasc.ZkController.register We are http://127.0.0.1:34889/collection1/ and leader is http://127.0.0.1:34889/collection1/
   [junit4]   2> 104282 T313 [control_collection] [shard1] [] []  oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:34889
   [junit4]   2> 104283 T313 [control_collection] [shard1] [] []  oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 104283 T313 [control_collection] [shard1] [] [collection1]  oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 104284 T313 [control_collection] [shard1] [] [collection1]  oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 104286 T321 [] [] [] []  oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 104289 T306 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:34889",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:34889_",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1"} current state version: 4
   [junit4]   2> 104290 T306 [] [] [] []  oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:34889",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:34889_",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1"}
   [junit4]   2> 104397 T322 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 104397 T321 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 106253 T279 [] [] [] []  oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/shard-1-001/cores/collection1
   [junit4]   2> 106256 T279 [] [] [] []  oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/shard-1-001
   [junit4]   2> 106257 T279 [] [] [] []  oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 106260 T279 [] [] [] []  oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:34897
   [junit4]   2> 106261 T279 [] [] [] []  oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=34896, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/shard-1-001/cores, solrconfig=solrconfig.xml, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/tempDir-001/jetty1, hostContext=/}
   [junit4]   2> 106261 T279 [] [] [] []  oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3972aa3f
   [junit4]   2> 106262 T279 [] [] [] []  oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/shard-1-001/'
   [junit4]   2> 106325 T279 [] [] [] []  oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/shard-1-001/solr.xml
   [junit4]   2> 106405 T279 [] [] [] []  oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/shard-1-001/cores
   [junit4]   2> 106406 T279 [] [] [] []  oasc.CoreContainer.<init> New CoreContainer 2096256924
   [junit4]   2> 106406 T279 [] [] [] []  oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/shard-1-001/]
   [junit4]   2> 106407 T279 [] [] [] []  oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/shard-1-001/lib
   [junit4]   2> 106408 T279 [] [] [] []  oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/shard-1-001/lib).
   [junit4]   2> 106421 T279 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 106422 T279 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 106422 T279 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 106423 T279 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 106423 T279 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 106423 T279 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 106424 T279 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 106424 T279 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 106424 T279 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 106425 T279 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 106425 T279 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 106426 T279 [] [] [] []  oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 106426 T279 [] [] [] []  oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 106427 T279 [] [] [] []  oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 106427 T279 [] [] [] []  oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 106427 T279 [] [] [] []  oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:34883/solr
   [junit4]   2> 106428 T279 [] [] [] []  oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 106428 T279 [] [] [] []  oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 106429 T279 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 106432 T334 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@343f32b9 name:ZooKeeperConnection Watcher:127.0.0.1:34883 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 106433 T279 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 106434 T279 [] [] [] []  oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 106438 T279 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 106440 T337 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@72964e43 name:ZooKeeperConnection Watcher:127.0.0.1:34883/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 106440 T279 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 106452 T279 [] [] [] []  oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 107505 T279 [] [] [] []  oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34896_
   [junit4]   2> 107506 T279 [] [] [] []  oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:34896_
   [junit4]   2> 107510 T279 [] [] [] []  oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 107517 T279 [] [] [] []  oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/shard-1-001/cores
   [junit4]   2> 107519 T279 [] [] [] []  oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/shard-1-001/cores/collection1/, shard=, collection=collection1, loadOnStartup=true, dataDir=data/, transient=false, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/shard-1-001/cores/collection1, coreNodeName=, config=solrconfig.xml}
   [junit4]   2> 107520 T279 [] [] [] []  oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/shard-1-001/cores/collection1/
   [junit4]   2> 107520 T279 [] [] [] []  oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 107524 T338 [collection1] [] [] [collection1]  oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 107524 T338 [collection1] [] [] [collection1]  oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 107527 T321 [] [] [] []  oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 107528 T338 [collection1] [] [] [collection1]  oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 107529 T306 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:34896",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:34896_",
   [junit4]   2> 	  "shard":null} current state version: 5
   [junit4]   2> 107529 T338 [collection1] [] [] [collection1]  oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 107529 T306 [] [] [] []  oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:34896",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:34896_",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 107530 T338 [collection1] [] [] [collection1]  oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 107530 T306 [] [] [] []  oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 107530 T306 [] [] [] []  oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 107533 T306 [] [] [] []  oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 107538 T338 [collection1] [] [] [collection1]  oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 107539 T338 [collection1] [] [] [collection1]  oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 107539 T338 [collection1] [] [] [collection1]  oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 107539 T338 [collection1] [] [] [collection1]  oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 107540 T338 [collection1] [] [] [collection1]  oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 107540 T338 [collection1] [] [] [collection1]  oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 107542 T338 [collection1] [] [] [collection1]  oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 107542 T338 [collection1] [] [] [collection1]  oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/shard-1-001/cores/collection1/'
   [junit4]   2> 107559 T338 [collection1] [] [] [collection1]  oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 107562 T338 [collection1] [] [] [collection1]  oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 107576 T338 [collection1] [] [] [collection1]  oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 107599 T338 [collection1] [] [] [collection1]  oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 107648 T338 [collection1] [] [] [collection1]  oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 107650 T338 [collection1] [] [] [collection1]  oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 107662 T338 [collection1] [] [] [collection1]  oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 108016 T338 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 108028 T338 [collection1] [] [] [collection1]  oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 108030 T338 [collection1] [] [] [collection1]  oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 108050 T338 [collection1] [] [] [collection1]  oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 108054 T338 [collection1] [] [] [collection1]  oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 108058 T338 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 108059 T338 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 108060 T338 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 108060 T338 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 108061 T338 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 108061 T338 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 108062 T338 [collection1] [] [] [collection1]  oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 108062 T338 [collection1] [] [] [collection1]  oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 108062 T338 [collection1] [] [] [collection1]  oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 108063 T338 [collection1] [] [] [collection1]  oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7ac28e11
   [junit4]   2> 108063 T338 [collection1] [] [] [collection1]  oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/shard-1-001/cores/collection1/data
   [junit4]   2> 108064 T338 [collection1] [] [] [collection1]  oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 108064 T338 [collection1] [] [] [collection1]  oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 108064 T338 [collection1] [] [] [collection1]  oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 212FDD5EC8B0F2CF-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 108064 T338 [collection1] [] [] [collection1]  oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=40, maxMergeAtOnceExplicit=41, maxMergedSegmentMB=77.2353515625, floorSegmentMB=0.7685546875, forceMergeDeletesPctAllowed=18.270899036203893, segmentsPerTier=21.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 108065 T338 [collection1] [] [] [collection1]  oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@42076ed1 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75fc7d34),segFN=segments_1,generation=1}
   [junit4]   2> 108065 T338 [collection1] [] [] [collection1]  oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 108072 T338 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 108072 T338 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 108073 T338 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 108073 T338 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 108073 T338 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 108073 T338 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 108074 T338 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 108074 T338 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 108075 T338 [collection1] [] [] [collection1]  oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 108087 T338 [collection1] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 108088 T338 [collection1] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 108090 T338 [collection1] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 108092 T338 [collection1] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 108098 T338 [collection1] [] [] [collection1]  oasc.RequestHandlers.initHandlersFromConfig Registered paths: standard,/update/csv,/admin/file,/admin/system,/admin/threads,/config,/update/json/docs,/admin/mbeans,/update/json,/admin/plugins,/admin/properties,/admin/segments,/schema,/admin/luke,/admin/ping,/update,/replication,/admin/logging,/get
   [junit4]   2> 108100 T338 [collection1] [] [] [collection1]  oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 108101 T338 [collection1] [] [] [collection1]  oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 108101 T338 [collection1] [] [] [collection1]  oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 108102 T338 [collection1] [] [] [collection1]  oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 108102 T338 [collection1] [] [] [collection1]  oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 108103 T338 [collection1] [] [] [collection1]  oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=10, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5845454432213795]
   [junit4]   2> 108104 T338 [collection1] [] [] [collection1]  oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@42076ed1 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75fc7d34),segFN=segments_1,generation=1}
   [junit4]   2> 108104 T338 [collection1] [] [] [collection1]  oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 108104 T338 [collection1] [] [] [collection1]  oass.SolrIndexSearcher.<init> Opening Searcher@1b583737[collection1] main
   [junit4]   2> 108104 T338 [collection1] [] [] [collection1]  oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 108106 T338 [collection1] [] [] [collection1]  oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 108106 T338 [collection1] [] [] [collection1]  oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 108107 T338 [collection1] [] [] [collection1]  oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 108107 T338 [collection1] [] [] [collection1]  oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 108107 T338 [collection1] [] [] [collection1]  oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 108108 T338 [collection1] [] [] [collection1]  oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 108108 T338 [collection1] [] [] [collection1]  oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 108109 T338 [collection1] [] [] [collection1]  oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 108109 T338 [collection1] [] [] [collection1]  oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 108110 T339 [collection1] [] [] [collection1]  oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1b583737[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 108111 T338 [collection1] [] [] [collection1]  oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 108112 T342 [collection1] [shard1] [] []  oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:34896 collection:collection1 shard:shard1
   [junit4]   2> 108112 T279 [] [] [] []  oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 108113 T279 [] [] [] []  oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 108113 T342 [collection1] [shard1] [] []  oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 108121 T342 [collection1] [shard1] [] []  oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 108124 T321 [] [] [] []  oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 108125 T342 [collection1] [shard1] [] []  oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 108125 T342 [collection1] [shard1] [] []  oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C26 name=collection1 org.apache.solr.core.SolrCore@5f972db9 url=http://127.0.0.1:34896/collection1 node=127.0.0.1:34896_ C26_STATE=coll:collection1 core:collection1 props:{core=collection1, base_url=http://127.0.0.1:34896, state=down, node_name=127.0.0.1:34896_}
   [junit4]   2> 108126 T342 [collection1] [shard1] [] []  C26 P34896 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:34896/collection1/
   [junit4]   2> 108126 T342 [collection1] [shard1] [] []  C26 P34896 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [j

[...truncated too long message...]

Proxy.close WARN Closing 0 connections to: http://127.0.0.1:45241/sgon/w
   [junit4]   2> 1661135 T6365 [control_collection] [shard1] [] [collection1]  oasc.SocketProxy.close WARN Closing 9 connections to: http://127.0.0.1:45163/sgon/w
   [junit4]   2> 1661135 T6365 [control_collection] [shard1] [] [collection1]  oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:45233/sgon/w
   [junit4]   2> 1661135 T6365 [control_collection] [shard1] [] [collection1]  oasc.SocketProxy.close WARN Closing 2 connections to: http://127.0.0.1:45189/sgon/w
   [junit4]   2> 1661135 T6365 [control_collection] [shard1] [] [collection1]  oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 1661136 T6365 [control_collection] [shard1] [] [collection1]  oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:45241/sgon/w
   [junit4]   2> 1661136 T6365 [control_collection] [shard1] [] [collection1]  oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:45163/sgon/w
   [junit4]   2> 1661136 T6365 [control_collection] [shard1] [] [collection1]  oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:45233/sgon/w
   [junit4]   2> 1661136 T6365 [control_collection] [shard1] [] [collection1]  oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:45189/sgon/w
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=212FDD5EC8B0F2CF -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ar_SA -Dtests.timezone=Africa/Ndjamena -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR    122s J3 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([212FDD5EC8B0F2CF:A97BE284664C9F37]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 1661182 T6365 [control_collection] [shard1] [] [collection1]  oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 212FDD5EC8B0F2CF-001
   [junit4]   2> 121863 T6364 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {timestamp=FST50, a_t=FST50, multiDefault=FSTOrd50, id=PostingsFormat(name=Memory doPackFST= false), range_facet_l=FST50, text=PostingsFormat(name=Memory doPackFST= true), _version_=FST50, intDefault=FST50}, docValues:{timestamp=DocValuesFormat(name=Memory)}, sim=RandomSimilarityProvider(queryNorm=false,coord=crazy): {}, locale=ar_SA, timezone=Africa/Ndjamena
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=129156200,total=404750336
   [junit4]   2> NOTE: All tests run in this JVM: [MultiThreadedOCPTest, TestFreeTextSuggestions, TestZkChroot, TestInfoStreamLogging, DistributedQueryComponentOptimizationTest, TestRecovery, TestMaxScoreQueryParser, PluginInfoTest, TestStressRecovery, TestExactStatsCache, TestFileDictionaryLookup, TestCustomSort, TestBinaryField, TestSolrQueryParser, UniqFieldsUpdateProcessorFactoryTest, TestManagedSchemaFieldResource, TestCloudManagedSchemaConcurrent, SuggesterFSTTest, AddBlockUpdateTest, TestPartialUpdateDeduplication, TestRangeQuery, RegexBoostProcessorTest, TestLuceneMatchVersion, TestOmitPositions, MigrateRouteKeyTest, ParsingFieldUpdateProcessorsTest, TestFieldResource, QueryElevationComponentTest, TestRandomFaceting, SuggesterWFSTTest, TestConfigSets, TestBulkSchemaAPI, BinaryUpdateRequestHandlerTest, ShardRoutingTest, TestInitQParser, FileBasedSpellCheckerTest, TestFastLRUCache, CurrencyFieldXmlFileTest, LeaderElectionIntegrationTest, TestQuerySenderListener, DateFieldTest, SignatureUpdateProcessorFactoryTest, RecoveryZkTest, TestAnalyzedSuggestions, TestNoOpRegenerator, SimpleFacetsTest, TestCloudInspectUtil, TermVectorComponentTest, TestConfigOverlay, ReturnFieldsTest, BlockCacheTest, CollectionsAPIAsyncDistributedZkTest, CircularListTest, TestDynamicFieldCollectionResource, TestCollationField, TestComplexPhraseQParserPlugin, ClusterStateUpdateTest, TestJsonFacets, BasicDistributedZk2Test, CollectionsAPIDistributedZkTest, BasicDistributedZkTest, ReplicationFactorTest]
   [junit4] Completed on J3 in 122.14s, 1 test, 1 error <<< FAILURES!

[...truncated 694 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 475 suites, 1894 tests, 3 errors, 62 ignored (22 assumptions)

Total time: 53 minutes 19 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 223094101 bytes
Compression is 0.0%
Took 45 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2830 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2830/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:65112/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:65112/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([F756D41A244A31BF:7F02EBC08AB65C47]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([F756D41A244A31BF:7F02EBC08AB65C47]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([F756D41A244A31BF:7F02EBC08AB65C47]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9452 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/init-core-data-001
   [junit4]   2> 136133 T284 [] [] [] []  oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /vd_/kd
   [junit4]   2> 136141 T284 [] [] [] []  oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 136142 T285 [] [] [] []  oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 136242 T284 [] [] [] []  oasc.ZkTestServer.run start zk server on port:55241
   [junit4]   2> 136243 T284 [] [] [] []  oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 136244 T284 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 136252 T292 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1b30660a name:ZooKeeperConnection Watcher:127.0.0.1:55241 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 136262 T284 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 136263 T284 [] [] [] []  oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 136264 T284 [] [] [] []  oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 136267 T284 [] [] [] []  oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 136268 T284 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 136270 T295 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@a2ce5c5 name:ZooKeeperConnection Watcher:127.0.0.1:55241/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 136270 T284 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 136271 T284 [] [] [] []  oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 136271 T284 [] [] [] []  oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 136274 T284 [] [] [] []  oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 136276 T284 [] [] [] []  oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 136278 T284 [] [] [] []  oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 136280 T284 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 136281 T284 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 136285 T284 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 136286 T284 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 136289 T284 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 136289 T284 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 136291 T284 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 136291 T284 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 136294 T284 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 136294 T284 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 136296 T284 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 136297 T284 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 136299 T284 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 136300 T284 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 136302 T284 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 136302 T284 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 136304 T284 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 136305 T284 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 136307 T284 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 136307 T284 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 136309 T284 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 136310 T284 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 137748 T284 [] [] [] []  oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/control-001/cores/collection1
   [junit4]   2> 137753 T284 [] [] [] []  oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 137757 T284 [] [] [] []  oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:34286
   [junit4]   2> 137758 T284 [] [] [] []  oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/tempDir-001/control/data, hostPort=45317, hostContext=/vd_/kd, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/control-001/cores}
   [junit4]   2> 137758 T284 [] [] [] []  oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 137759 T284 [] [] [] []  oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/control-001/'
   [junit4]   2> 137803 T284 [] [] [] []  oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/control-001/solr.xml
   [junit4]   2> 137861 T284 [] [] [] []  oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/control-001/cores
   [junit4]   2> 137862 T284 [] [] [] []  oasc.CoreContainer.<init> New CoreContainer 1371813641
   [junit4]   2> 137862 T284 [] [] [] []  oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/control-001/]
   [junit4]   2> 137862 T284 [] [] [] []  oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/control-001/lib
   [junit4]   2> 137863 T284 [] [] [] []  oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/control-001/lib).
   [junit4]   2> 137874 T284 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 137874 T284 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 137874 T284 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 137874 T284 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 137875 T284 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 137875 T284 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 137875 T284 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 137876 T284 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 137876 T284 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 137876 T284 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 137876 T284 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 137877 T284 [] [] [] []  oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 137877 T284 [] [] [] []  oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 137878 T284 [] [] [] []  oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 137878 T284 [] [] [] []  oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 137878 T284 [] [] [] []  oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:55241/solr
   [junit4]   2> 137878 T284 [] [] [] []  oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 137879 T284 [] [] [] []  oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 137880 T284 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 137882 T307 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@38cd4b86 name:ZooKeeperConnection Watcher:127.0.0.1:55241 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 137882 T284 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 137883 T284 [] [] [] []  oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 137885 T284 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 137887 T310 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@712545be name:ZooKeeperConnection Watcher:127.0.0.1:55241/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 137887 T284 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 137889 T284 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 137891 T284 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 137893 T284 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 137894 T284 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 137896 T284 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 137898 T284 [] [] [] []  oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 137900 T284 [] [] [] []  oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 137901 T284 [] [] [] []  oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 137902 T284 [] [] [] []  oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45317_vd_%2Fkd
   [junit4]   2> 137903 T284 [] [] [] []  oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:45317_vd_%2Fkd
   [junit4]   2> 137904 T284 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 137905 T284 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 137907 T284 [] [] [] []  oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 137908 T284 [] [] [] []  oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:45317_vd_%2Fkd
   [junit4]   2> 137909 T284 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 137910 T284 [] [] [] []  oasc.Overseer.start Overseer (id=93537663417581571-127.0.0.1:45317_vd_%2Fkd-n_0000000000) starting
   [junit4]   2> 137912 T284 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 137917 T284 [] [] [] []  oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 137918 T312 [] [] [] []  oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 137918 T284 [] [] [] []  oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 137918 T311 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 137922 T284 [] [] [] []  oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/control-001/cores
   [junit4]   2> 137924 T284 [] [] [] []  oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {dataDir=data/, transient=false, collection=control_collection, config=solrconfig.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/control-001/cores/collection1/, name=collection1, coreNodeName=, shard=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/control-001/cores/collection1, loadOnStartup=true, schema=schema.xml}
   [junit4]   2> 137925 T284 [] [] [] []  oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/control-001/cores/collection1/
   [junit4]   2> 137925 T284 [] [] [] []  oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 137926 T314 [control_collection] [] [] [collection1]  oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 137926 T314 [control_collection] [] [] [collection1]  oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 137928 T310 [] [] [] []  oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 137928 T314 [control_collection] [] [] [collection1]  oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 137929 T311 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:45317_vd_%2Fkd",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:45317/vd_/kd"} current state version: 0
   [junit4]   2> 137929 T311 [] [] [] []  oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:45317_vd_%2Fkd",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:45317/vd_/kd"}
   [junit4]   2> 137930 T311 [] [] [] []  oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 137930 T311 [] [] [] []  oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 137931 T310 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 138928 T314 [control_collection] [] [] [collection1]  oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 138929 T314 [control_collection] [] [] [collection1]  oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 138929 T314 [control_collection] [] [] [collection1]  oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 138930 T314 [control_collection] [] [] [collection1]  oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 138930 T314 [control_collection] [] [] [collection1]  oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 138931 T314 [control_collection] [] [] [collection1]  oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/control-001/cores/collection1/'
   [junit4]   2> 138945 T314 [control_collection] [] [] [collection1]  oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 138947 T314 [control_collection] [] [] [collection1]  oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 138958 T314 [control_collection] [] [] [collection1]  oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 138979 T314 [control_collection] [] [] [collection1]  oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 139024 T314 [control_collection] [] [] [collection1]  oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 139026 T314 [control_collection] [] [] [collection1]  oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 139039 T314 [control_collection] [] [] [collection1]  oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 139409 T314 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 139426 T314 [control_collection] [] [] [collection1]  oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 139429 T314 [control_collection] [] [] [collection1]  oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 139443 T314 [control_collection] [] [] [collection1]  oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 139450 T314 [control_collection] [] [] [collection1]  oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 139455 T314 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 139456 T314 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 139456 T314 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 139457 T314 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 139458 T314 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 139458 T314 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 139458 T314 [control_collection] [] [] [collection1]  oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 139459 T314 [control_collection] [] [] [collection1]  oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 139459 T314 [control_collection] [] [] [collection1]  oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/control-001/cores/collection1/data/
   [junit4]   2> 139460 T314 [control_collection] [] [] [collection1]  oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7c23ea0c
   [junit4]   2> 139461 T314 [control_collection] [] [] [collection1]  oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/control-001/cores/collection1/data
   [junit4]   2> 139461 T314 [control_collection] [] [] [collection1]  oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/control-001/cores/collection1/data/index/
   [junit4]   2> 139461 T314 [control_collection] [] [] [collection1]  oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 139462 T314 [control_collection] [] [] [collection1]  oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/control-001/cores/collection1/data/index
   [junit4]   2> 139462 T314 [control_collection] [] [] [collection1]  oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=10, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 139464 T314 [control_collection] [] [] [collection1]  oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2425bc73 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2977348f),segFN=segments_1,generation=1}
   [junit4]   2> 139464 T314 [control_collection] [] [] [collection1]  oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 139472 T314 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 139472 T314 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 139473 T314 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 139473 T314 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 139474 T314 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 139474 T314 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 139475 T314 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 139475 T314 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 139475 T314 [control_collection] [] [] [collection1]  oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 139506 T314 [control_collection] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 139509 T314 [control_collection] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 139512 T314 [control_collection] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 139515 T314 [control_collection] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 139522 T314 [control_collection] [] [] [collection1]  oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update/csv,/admin/plugins,/admin/luke,/update/json,/admin/logging,standard,/admin/file,/admin/threads,/admin/properties,/admin/system,/schema,/get,/update,/admin/ping,/update/json/docs,/admin/mbeans,/admin/segments,/replication,/config
   [junit4]   2> 139525 T314 [control_collection] [] [] [collection1]  oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 139527 T314 [control_collection] [] [] [collection1]  oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 139527 T314 [control_collection] [] [] [collection1]  oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 139529 T314 [control_collection] [] [] [collection1]  oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 139530 T314 [control_collection] [] [] [collection1]  oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 139531 T314 [control_collection] [] [] [collection1]  oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=28, maxMergeAtOnceExplicit=18, maxMergedSegmentMB=38.3330078125, floorSegmentMB=0.90234375, forceMergeDeletesPctAllowed=15.808509722862036, segmentsPerTier=11.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 139532 T314 [control_collection] [] [] [collection1]  oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2425bc73 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2977348f),segFN=segments_1,generation=1}
   [junit4]   2> 139532 T314 [control_collection] [] [] [collection1]  oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 139533 T314 [control_collection] [] [] [collection1]  oass.SolrIndexSearcher.<init> Opening Searcher@41532b15[collection1] main
   [junit4]   2> 139533 T314 [control_collection] [] [] [collection1]  oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 139534 T314 [control_collection] [] [] [collection1]  oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 139534 T314 [control_collection] [] [] [collection1]  oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 139535 T314 [control_collection] [] [] [collection1]  oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 139535 T314 [control_collection] [] [] [collection1]  oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 139535 T314 [control_collection] [] [] [collection1]  oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 139536 T314 [control_collection] [] [] [collection1]  oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 139536 T314 [control_collection] [] [] [collection1]  oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 139537 T314 [control_collection] [] [] [collection1]  oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 139537 T314 [control_collection] [] [] [collection1]  oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 139538 T315 [control_collection] [] [] [collection1]  oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@41532b15[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 139539 T314 [control_collection] [] [] [collection1]  oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 139540 T318 [control_collection] [shard1] [] []  oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:45317/vd_/kd collection:control_collection shard:shard1
   [junit4]   2> 139540 T284 [] [] [] []  oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 139541 T284 [] [] [] []  oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 139541 T318 [control_collection] [shard1] [] []  oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 139545 T284 [] [] [] []  oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 139546 T284 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 139547 T318 [control_collection] [shard1] [] []  oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 139547 T322 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7dbfbfd name:ZooKeeperConnection Watcher:127.0.0.1:55241/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 139548 T284 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 139549 T284 [] [] [] []  oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 139549 T318 [control_collection] [shard1] [] []  oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 139549 T310 [] [] [] []  oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 139549 T318 [control_collection] [shard1] [] []  oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 139549 T284 [] [] [] []  oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 139550 T311 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C72 name=collection1 org.apache.solr.core.SolrCore@420c9bfa url=http://127.0.0.1:45317/vd_/kd/collection1 node=127.0.0.1:45317_vd_%2Fkd C72_STATE=coll:control_collection core:collection1 props:{state=down, core=collection1, node_name=127.0.0.1:45317_vd_%2Fkd, base_url=http://127.0.0.1:45317/vd_/kd}
   [junit4]   2> 139550 T318 [control_collection] [shard1] [] []  C72 P45317 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:45317/vd_/kd/collection1/
   [junit4]   2> 139552 T318 [control_collection] [shard1] [] []  C72 P45317 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 139552 T318 [control_collection] [shard1] [] []  C72 P45317 oasc.SyncStrategy.syncToMe http://127.0.0.1:45317/vd_/kd/collection1/ has no replicas
   [junit4]   2> 139552 T318 [control_collection] [shard1] [] []  oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:45317/vd_/kd/collection1/ shard1
   [junit4]   2> 139553 T310 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 139553 T322 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 139553 T284 [] [] [] []  oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 139553 T318 [control_collection] [shard1] [] []  oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 139554 T284 [] [] [] []  oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 139555 T284 [] [] [] []  oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 139556 T284 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 139557 T325 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@308f8272 name:ZooKeeperConnection Watcher:127.0.0.1:55241/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 139558 T284 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 139559 T310 [] [] [] []  oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 139559 T284 [] [] [] []  oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 139560 T311 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:45317/vd_/kd",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 139563 T311 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 139563 T311 [] [] [] []  oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 139564 T322 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 139564 T310 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 139567 T310 [] [] [] []  oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 139568 T311 [] [] [] []  oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 139569 T310 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 139569 T322 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 139610 T318 [control_collection] [shard1] [] []  oasc.ZkController.register We are http://127.0.0.1:45317/vd_/kd/collection1/ and leader is http://127.0.0.1:45317/vd_/kd/collection1/
   [junit4]   2> 139611 T318 [control_collection] [shard1] [] []  oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:45317/vd_/kd
   [junit4]   2> 139611 T318 [control_collection] [shard1] [] []  oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 139611 T318 [control_collection] [shard1] [] [collection1]  oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 139612 T318 [control_collection] [shard1] [] [collection1]  oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 139613 T310 [] [] [] []  oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 139614 T311 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:45317_vd_%2Fkd",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:45317/vd_/kd"} current state version: 4
   [junit4]   2> 139615 T311 [] [] [] []  oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:45317_vd_%2Fkd",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:45317/vd_/kd"}
   [junit4]   2> 139719 T310 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 139719 T322 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 141005 T284 [] [] [] []  oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/shard-1-001/cores/collection1
   [junit4]   2> 141008 T284 [] [] [] []  oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/shard-1-001
   [junit4]   2> 141009 T284 [] [] [] []  oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 141011 T284 [] [] [] []  oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:10929
   [junit4]   2> 141011 T284 [] [] [] []  oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/shard-1-001/cores, hostContext=/vd_/kd, hostPort=45546, solrconfig=solrconfig.xml, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/tempDir-001/jetty1}
   [junit4]   2> 141012 T284 [] [] [] []  oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 141012 T284 [] [] [] []  oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/shard-1-001/'
   [junit4]   2> 141049 T284 [] [] [] []  oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/shard-1-001/solr.xml
   [junit4]   2> 141128 T284 [] [] [] []  oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/shard-1-001/cores
   [junit4]   2> 141128 T284 [] [] [] []  oasc.CoreContainer.<init> New CoreContainer 601335624
   [junit4]   2> 141129 T284 [] [] [] []  oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/shard-1-001/]
   [junit4]   2> 141129 T284 [] [] [] []  oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/shard-1-001/lib
   [junit4]   2> 141130 T284 [] [] [] []  oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/shard-1-001/lib).
   [junit4]   2> 141140 T284 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 141140 T284 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 141140 T284 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 141141 T284 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 141141 T284 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 141141 T284 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 141142 T284 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 141142 T284 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 141142 T284 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 141142 T284 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 141143 T284 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 141143 T284 [] [] [] []  oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 141144 T284 [] [] [] []  oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 141144 T284 [] [] [] []  oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 141144 T284 [] [] [] []  oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 141145 T284 [] [] [] []  oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:55241/solr
   [junit4]   2> 141145 T284 [] [] [] []  oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 141145 T284 [] [] [] []  oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 141146 T284 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 141148 T337 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@63b3bb0a name:ZooKeeperConnection Watcher:127.0.0.1:55241 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 141149 T284 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 141150 T284 [] [] [] []  oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 141152 T284 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 141154 T340 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@fc54456 name:ZooKeeperConnection Watcher:127.0.0.1:55241/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 141154 T284 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 141158 T284 [] [] [] []  oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 142164 T284 [] [] [] []  oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45546_vd_%2Fkd
   [junit4]   2> 142165 T284 [] [] [] []  oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:45546_vd_%2Fkd
   [junit4]   2> 142168 T284 [] [] [] []  oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 142170 T284 [] [] [] []  oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/shard-1-001/cores
   [junit4]   2> 142173 T284 [] [] [] []  oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {dataDir=data/, name=collection1, loadOnStartup=true, shard=, coreNodeName=, schema=schema.xml, config=solrconfig.xml, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/shard-1-001/cores/collection1, collection=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/shard-1-001/cores/collection1/}
   [junit4]   2> 142174 T284 [] [] [] []  oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/shard-1-001/cores/collection1/
   [junit4]   2> 142174 T284 [] [] [] []  oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 142177 T341 [collection1] [] [] [collection1]  oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 142177 T341 [collection1] [] [] [collection1]  oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 142179 T310 [] [] [] []  oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 142179 T341 [collection1] [] [] [collection1]  oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 142180 T341 [collection1] [] [] [collection1]  oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 142180 T311 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:45546_vd_%2Fkd",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:45546/vd_/kd"} current state version: 5
   [junit4]   2> 142180 T341 [collection1] [] [] [collection1]  oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 142180 T311 [] [] [] []  oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:45546_vd_%2Fkd",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:45546/vd_/kd"}
   [junit4]   2> 142181 T311 [] [] [] []  oasco.ReplicaMutator.updateState Collection already exists with numShards=3
   [junit4]   2> 142181 T311 [] [] [] []  oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 142181 T341 [collection1] [] [] [collection1]  oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 142181 T341 [collection1] [] [] [collection1]  oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 142182 T311 [] [] [] []  oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 142183 T340 [] [] [] []  oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 142184 T340 [] [] [] []  oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 143182 T341 [collection1] [] [] [collection1]  oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 143183 T341 [collection1] [] [] [collection1]  oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 143184 T341 [collection1] [] [] [collection1]  oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 143185 T341 [collection1] [] [] [collection1]  oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 143186 T341 [collection1] [] [] [collection1]  oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 143186 T341 [collection1] [] [] [collection1]  oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/shard-1-001/cores/collection1/'
   [junit4]   2> 143215 T341 [collection1] [] [] [collection1]  oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 143219 T341 [collection1] [] [] [collection1]  oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 143245 T341 [collection1] [] [] [collection1]  oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 143291 T341 [collection1] [] [] [collection1]  oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 143381 T341 [collection1] [] [] [collection1]  oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 143383 T341 [collection1] [] [] [collection1]  oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 143412 T341 [collection1] [] [] [collection1]  oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 144048 T341 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 144069 T341 [collection1] [] [] [collection1]  oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 144073 T341 [collection1] [] [] [collection1]  oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 144095 T341 [collection1] [] [] [collection1]  oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 144101 T341 [collection1] [] [] [collection1]  oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 144107 T341 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 144108 T341 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 144109 T341 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 144109 T341 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 144110 T341 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 144110 T341 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 144111 T341 [collection1] [] [] [collection1]  oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 144111 T341 [collection1] [] [] [collection1]  oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 144112 T341 [collection1] [] [] [collection1]  oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 144112 T341 [collection1] [] [] [collection1]  oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7c23ea0c
   [junit4]   2> 144113 T341 [collection1] [] [] [collection1]  oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/shard-1-001/cores/collection1/data
   [junit4]   2> 144114 T341 [collection1] [] [] [collection1]  oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 144114 T341 [collection1] [] [] [collection1]  oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 144115 T341 [collection1] [] [] [collection1]  oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F756D41A244A31BF-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 144115 T341 [collection1] [] [] [collection1]  oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=10, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 144116 T341 [collection1] [] [] [collection1]  oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2670c674 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@35f4fd63),segFN=segments_1,generation=1}
   [junit4]   2> 144117 T341 [collection1] [] [] [collection1]  oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 144127 T341 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 144128 T341 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 144128 T341 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 144128 T341 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 144129 T341 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 144129 T341 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 144130 T341 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 144130 T341 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 144131 T341 [collection1] [] [] [collection1]  oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 144160 T341 [collection1] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 144163 T341 [collection1] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 144166 T341 [collection1] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 144169 T341 [collection1] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 144179 T341 [collection1] [] [] [collection1]  oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update/csv,/admin/plugins,/admin/luke,/update/json,/admin/logging,standard,/admin/file,/admin/threads,/admin/properties,/admin/system,/schema,/get,/update,/admin/ping,/update/json/docs,/admin/mbeans,/admin/segments,/replication,/config
   [junit4]   2> 144182 T341 [collection1] [] [] [collection1]  oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 144185 T341 [collection1] [] [] [collection1]  oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 144185 T341 [collection1] [] [] [collection1]  oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 144187 T341 [collection1] [] [] [collection1]  oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 144188 T341 [collection1] [] [] [collection1]  oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 144189 T341 [collection1] [] [] [collection1]  oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=28, maxMergeAtOnceExplicit=18, maxMergedSegmentMB=38.3330078125, floorSegmentMB=0.90234375, forceMergeDeletesPctAllowed=15.808509722862036, segmentsPerTier=11.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 144190 T341 [collection1] [] [] [collection1]  oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2670c674 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@35f4fd63),segFN=segments_1,generation=1}
   [junit4]   2> 144190 T341 [collection1] [] [] [collection1]  oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 144191 T341 [collection1] [] [] [collection1]  oass.SolrIndexSearcher.<init> Opening Searcher@5e5909ed[collection1] main
   [junit4]   2> 144191 T341 [collection1] [] [] [collection1]  oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 144192 T341 [collection1] [] [] [collection1]  oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 144193 T341 [collection1] [] [] [collection1]  oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 144193 T341 [collection1] [] [] [collection1]  oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 144193 T341 [collection1] [] [] [collection1]  oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 144194 T341 [collection1] [] [] [collection1]  oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 144194 T341 [collection1] [] [] [collection1]  oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 144195 T341 [collection1] [] [] [collection1]  oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 144195 T341 [collection1] [] [] [collection1]  oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 144196 T341 [collection1] [] [] [collection1]  oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 144197 T342 [collection1] [] [] [collection1]  oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5e5909ed[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 144198 T341 [collection1] [] [] [collection1]  oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 144198 T345 [collection1] [shard1] [] []  oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:45546/vd_/kd collection:collection1 shard:shard1
   [junit4]   2> 144199 T284 [] [] [] []  oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 144200 T284 [] [] [] []  oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 144200 T345 [collection1] [shard1] [] []  oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 144205 T345 [collection1] [shard1] [] []  oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 144207 T310 [] [] [] []  oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 144207 T345 [collection1] [shard1] [] []  oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 144208 T345 [collection1] [shard1] [] []  oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C73 name=collection1 org.apache.solr.core.SolrCore@373cb761 url=http://127.0.0.1:45546/vd_/kd/collection1 node=127.0.0.1:45546_vd_%2Fkd C73_STATE=coll:collection1 core:collection1 props:{state=down, core=collection

[...truncated too long message...]

ta
   [junit4]   2> 1520938 T3562 [control_collection] [shard1] [] [collection1]  oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> 1521039 T3562 [control_collection] [shard1] [] [collection1]  oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:65236 65236
   [junit4]   2> 1521263 T3563 [] [] [] []  oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:65236 65236
   [junit4]   2> 1521265 T3563 [] [] [] []  oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 		2	/solr/collections/collection1/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1521265 T3562 [control_collection] [shard1] [] [collection1]  oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:65268/
   [junit4]   2> 1521266 T3562 [control_collection] [shard1] [] [collection1]  oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:65244/
   [junit4]   2> 1521266 T3562 [control_collection] [shard1] [] [collection1]  oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:65260/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=F756D41A244A31BF -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=sq -Dtests.timezone=GMT0 -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   66.7s J1 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([F756D41A244A31BF:7F02EBC08AB65C47]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 1521285 T3562 [control_collection] [shard1] [] [collection1]  oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest F756D41A244A31BF-001
   [junit4]   2> 66725 T3561 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarityProvider(queryNorm=true,coord=crazy): {}, locale=sq, timezone=GMT0
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=31644344,total=311951360
   [junit4]   2> NOTE: All tests run in this JVM: [TestReplicationHandlerBackup, AnalyticsMergeStrategyTest, MultiTermTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, ChaosMonkeyNothingIsSafeTest, TestPHPSerializedResponseWriter, ZkNodePropsTest, TestCollectionAPI, OpenCloseCoreStressTest, AliasIntegrationTest, RAMDirectoryFactoryTest, SearchHandlerTest, TestDynamicFieldResource, LoggingHandlerTest, SchemaVersionSpecificBehaviorTest, TestWriterPerf, TestIntervalFaceting, TestZkChroot, FullSolrCloudDistribCmdsTest, URLClassifyProcessorTest, CacheHeaderTest, HdfsLockFactoryTest, DistribCursorPagingTest, ClusterStateUpdateTest, TestSolrDeletionPolicy2, SortByFunctionTest, DistributedMLTComponentTest, VMParamsZkACLAndCredentialsProvidersTest, SliceStateTest, ShowFileRequestHandlerTest, TestLeaderElectionZkExpiry, TestSolrQueryParser, TestFoldingMultitermQuery, DistributedQueryComponentOptimizationTest, TestDefaultStatsCache, TestFieldTypeCollectionResource, TestDistributedMissingSort, TestNoOpRegenerator, DocValuesMissingTest, TestCopyFieldCollectionResource, AtomicUpdatesTest, TermVectorComponentTest, TestInitQParser, TestPivotHelperCode, TestAnalyzedSuggestions, AnalyticsQueryTest, HdfsDirectoryFactoryTest, EnumFieldTest, TimeZoneUtilsTest, TestConfigSets, TestMaxScoreQueryParser, TestRandomFaceting, TestSolrQueryParserDefaultOperatorResource, TestBinaryResponseWriter, TestDefaultSimilarityFactory, TestJmxMonitoredMap, SaslZkACLProviderTest, TestReqParamsAPI, BlockCacheTest, HttpPartitionTest, MultiThreadedOCPTest, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J1 in 67.01s, 1 test, 1 error <<< FAILURES!

[...truncated 701 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 475 suites, 1894 tests, 3 errors, 62 ignored (22 assumptions)

Total time: 54 minutes 26 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 206308104 bytes
Compression is 0.0%
Took 40 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2829 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2829/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:19971/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19971/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([7C1B43108DE3652A:F44F7CCA231F08D2]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:16006/hk/ho/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:16006/hk/ho/collection1
	at __randomizedtesting.SeedInfo.seed([7C1B43108DE3652A:F44F7CCA231F08D2]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:16006/hk/ho/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:20165/oelk/k/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:20165/oelk/k/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([7C1B43108DE3652A:F44F7CCA231F08D2]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:20165/oelk/k/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 53 more




Build Log:
[...truncated 9483 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/init-core-data-001
   [junit4]   2> 225412 T637 [] [] [] []  oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 225418 T637 [] [] [] []  oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 225419 T638 [] [] [] []  oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 225518 T637 [] [] [] []  oasc.ZkTestServer.run start zk server on port:19950
   [junit4]   2> 225519 T637 [] [] [] []  oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 225520 T637 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 225523 T645 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1b27c003 name:ZooKeeperConnection Watcher:127.0.0.1:19950 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 225523 T637 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 225523 T637 [] [] [] []  oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 225524 T637 [] [] [] []  oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 225525 T637 [] [] [] []  oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 225526 T637 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 225528 T648 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1b948a84 name:ZooKeeperConnection Watcher:127.0.0.1:19950/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 225528 T637 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 225528 T637 [] [] [] []  oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 225528 T637 [] [] [] []  oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 225530 T637 [] [] [] []  oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 225531 T637 [] [] [] []  oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 225532 T637 [] [] [] []  oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 225533 T637 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 225533 T637 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 225535 T637 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 225536 T637 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 225637 T637 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 225637 T637 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 225639 T637 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 225639 T637 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 225641 T637 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 225641 T637 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 225642 T637 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 225643 T637 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 225644 T637 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 225645 T637 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 225646 T637 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 225646 T637 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 225648 T637 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 225648 T637 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 225649 T637 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 225650 T637 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 225651 T637 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 225652 T637 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 226826 T637 [] [] [] []  oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/control-001/cores/collection1
   [junit4]   2> 226830 T637 [] [] [] []  oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 226833 T637 [] [] [] []  oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:19959
   [junit4]   2> 226833 T637 [] [] [] []  oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/control-001/cores, hostContext=/, hostPort=19958}
   [junit4]   2> 226834 T637 [] [] [] []  oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 226834 T637 [] [] [] []  oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/control-001/'
   [junit4]   2> 226864 T637 [] [] [] []  oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/control-001/solr.xml
   [junit4]   2> 226900 T637 [] [] [] []  oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/control-001/cores
   [junit4]   2> 226901 T637 [] [] [] []  oasc.CoreContainer.<init> New CoreContainer 95387101
   [junit4]   2> 226901 T637 [] [] [] []  oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/control-001/]
   [junit4]   2> 226901 T637 [] [] [] []  oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/control-001/lib
   [junit4]   2> 226902 T637 [] [] [] []  oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/control-001/lib).
   [junit4]   2> 226911 T637 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 226911 T637 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 226911 T637 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 226912 T637 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 226912 T637 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 226912 T637 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 226912 T637 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 226912 T637 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 226913 T637 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 226913 T637 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 226913 T637 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 226914 T637 [] [] [] []  oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 226914 T637 [] [] [] []  oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 226914 T637 [] [] [] []  oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 226914 T637 [] [] [] []  oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 226915 T637 [] [] [] []  oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:19950/solr
   [junit4]   2> 226915 T637 [] [] [] []  oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 226915 T637 [] [] [] []  oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 226916 T637 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 226918 T660 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@bf44e3d name:ZooKeeperConnection Watcher:127.0.0.1:19950 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 226918 T637 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 226918 T637 [] [] [] []  oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 226920 T637 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 226921 T663 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@573a3e1a name:ZooKeeperConnection Watcher:127.0.0.1:19950/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 226921 T637 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 226922 T637 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 226924 T637 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 226926 T637 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 226927 T637 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 226929 T637 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 226930 T637 [] [] [] []  oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 226931 T637 [] [] [] []  oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 226932 T637 [] [] [] []  oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 226933 T637 [] [] [] []  oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:19958_
   [junit4]   2> 226934 T637 [] [] [] []  oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:19958_
   [junit4]   2> 226935 T637 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 226936 T637 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 226937 T637 [] [] [] []  oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 226938 T637 [] [] [] []  oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:19958_
   [junit4]   2> 226938 T637 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 226939 T637 [] [] [] []  oasc.Overseer.start Overseer (id=93537188307140611-127.0.0.1:19958_-n_0000000000) starting
   [junit4]   2> 226941 T637 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 226945 T637 [] [] [] []  oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 226945 T637 [] [] [] []  oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 226946 T664 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 226945 T665 [] [] [] []  oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 226948 T637 [] [] [] []  oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/control-001/cores
   [junit4]   2> 226950 T637 [] [] [] []  oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {dataDir=data/, loadOnStartup=true, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/control-001/cores/collection1/, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/control-001/cores/collection1, collection=control_collection, shard=, config=solrconfig.xml, transient=false, coreNodeName=, name=collection1}
   [junit4]   2> 226950 T637 [] [] [] []  oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/control-001/cores/collection1/
   [junit4]   2> 226951 T637 [] [] [] []  oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 226951 T667 [control_collection] [] [] [collection1]  oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 226951 T667 [control_collection] [] [] [collection1]  oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 226952 T667 [control_collection] [] [] [collection1]  oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 226952 T663 [] [] [] []  oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 226953 T664 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:19958",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:19958_"} current state version: 0
   [junit4]   2> 226953 T664 [] [] [] []  oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:19958",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:19958_"}
   [junit4]   2> 226953 T664 [] [] [] []  oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 226954 T664 [] [] [] []  oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 226955 T663 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 227952 T667 [control_collection] [] [] [collection1]  oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 227953 T667 [control_collection] [] [] [collection1]  oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 227953 T667 [control_collection] [] [] [collection1]  oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 227953 T667 [control_collection] [] [] [collection1]  oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 227954 T667 [control_collection] [] [] [collection1]  oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 227954 T667 [control_collection] [] [] [collection1]  oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/control-001/cores/collection1/'
   [junit4]   2> 227967 T667 [control_collection] [] [] [collection1]  oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 227968 T667 [control_collection] [] [] [collection1]  oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 227976 T667 [control_collection] [] [] [collection1]  oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 227992 T667 [control_collection] [] [] [collection1]  oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 228027 T667 [control_collection] [] [] [collection1]  oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 228128 T667 [control_collection] [] [] [collection1]  oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 228135 T667 [control_collection] [] [] [collection1]  oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 228412 T667 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 228423 T667 [control_collection] [] [] [collection1]  oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 228425 T667 [control_collection] [] [] [collection1]  oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 228435 T667 [control_collection] [] [] [collection1]  oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 228438 T667 [control_collection] [] [] [collection1]  oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 228441 T667 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 228442 T667 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 228442 T667 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 228442 T667 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 228442 T667 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 228443 T667 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 228443 T667 [control_collection] [] [] [collection1]  oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 228443 T667 [control_collection] [] [] [collection1]  oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 228443 T667 [control_collection] [] [] [collection1]  oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/control-001/cores/collection1/data/
   [junit4]   2> 228443 T667 [control_collection] [] [] [collection1]  oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3cd74bcf
   [junit4]   2> 228444 T667 [control_collection] [] [] [collection1]  oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/control-001/cores/collection1/data
   [junit4]   2> 228444 T667 [control_collection] [] [] [collection1]  oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/control-001/cores/collection1/data/index/
   [junit4]   2> 228445 T667 [control_collection] [] [] [collection1]  oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 228445 T667 [control_collection] [] [] [collection1]  oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/control-001/cores/collection1/data/index
   [junit4]   2> 228445 T667 [control_collection] [] [] [collection1]  oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=45, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6125429853411988]
   [junit4]   2> 228446 T667 [control_collection] [] [] [collection1]  oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@414886ea lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@918e22),segFN=segments_1,generation=1}
   [junit4]   2> 228446 T667 [control_collection] [] [] [collection1]  oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 228451 T667 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 228452 T667 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 228452 T667 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 228452 T667 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 228452 T667 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 228452 T667 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 228453 T667 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 228453 T667 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 228453 T667 [control_collection] [] [] [collection1]  oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 228465 T667 [control_collection] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 228467 T667 [control_collection] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 228468 T667 [control_collection] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 228470 T667 [control_collection] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 228474 T667 [control_collection] [] [] [collection1]  oasc.RequestHandlers.initHandlersFromConfig Registered paths: /config,/admin/logging,/admin/file,/admin/plugins,standard,/update,/admin/segments,/update/json/docs,/admin/threads,/admin/ping,/replication,/admin/system,/get,/admin/mbeans,/schema,/update/json,/admin/luke,/update/csv,/admin/properties
   [junit4]   2> 228476 T667 [control_collection] [] [] [collection1]  oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 228477 T667 [control_collection] [] [] [collection1]  oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 228477 T667 [control_collection] [] [] [collection1]  oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 228478 T667 [control_collection] [] [] [collection1]  oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 228478 T667 [control_collection] [] [] [collection1]  oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 228479 T667 [control_collection] [] [] [collection1]  oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=38, maxMergeAtOnceExplicit=27, maxMergedSegmentMB=73.0263671875, floorSegmentMB=1.0185546875, forceMergeDeletesPctAllowed=2.974923965248074, segmentsPerTier=47.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 228479 T667 [control_collection] [] [] [collection1]  oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@414886ea lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@918e22),segFN=segments_1,generation=1}
   [junit4]   2> 228479 T667 [control_collection] [] [] [collection1]  oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 228480 T667 [control_collection] [] [] [collection1]  oass.SolrIndexSearcher.<init> Opening Searcher@446e1912[collection1] main
   [junit4]   2> 228480 T667 [control_collection] [] [] [collection1]  oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 228481 T667 [control_collection] [] [] [collection1]  oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 228481 T667 [control_collection] [] [] [collection1]  oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 228481 T667 [control_collection] [] [] [collection1]  oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 228481 T667 [control_collection] [] [] [collection1]  oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 228481 T667 [control_collection] [] [] [collection1]  oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 228482 T667 [control_collection] [] [] [collection1]  oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 228482 T667 [control_collection] [] [] [collection1]  oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 228482 T667 [control_collection] [] [] [collection1]  oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 228482 T667 [control_collection] [] [] [collection1]  oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 228483 T668 [control_collection] [] [] [collection1]  oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@446e1912[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 228484 T667 [control_collection] [] [] [collection1]  oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 228485 T671 [control_collection] [shard1] [] []  oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:19958 collection:control_collection shard:shard1
   [junit4]   2> 228485 T637 [] [] [] []  oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 228485 T637 [] [] [] []  oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 228485 T671 [control_collection] [shard1] [] []  oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 228489 T637 [] [] [] []  oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 228489 T671 [control_collection] [shard1] [] []  oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 228489 T637 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 228491 T675 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@23de4005 name:ZooKeeperConnection Watcher:127.0.0.1:19950/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 228491 T637 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 228491 T671 [control_collection] [shard1] [] []  oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 228491 T637 [] [] [] []  oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 228491 T671 [control_collection] [shard1] [] []  oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 228491 T663 [] [] [] []  oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> ASYNC  NEW_CORE C119 name=collection1 org.apache.solr.core.SolrCore@4b0886c3 url=http://127.0.0.1:19958/collection1 node=127.0.0.1:19958_ C119_STATE=coll:control_collection core:collection1 props:{core=collection1, state=down, base_url=http://127.0.0.1:19958, node_name=127.0.0.1:19958_}
   [junit4]   2> 228492 T671 [control_collection] [shard1] [] []  C119 P19958 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:19958/collection1/
   [junit4]   2> 228492 T637 [] [] [] []  oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 228493 T664 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 228493 T671 [control_collection] [shard1] [] []  C119 P19958 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 228494 T671 [control_collection] [shard1] [] []  C119 P19958 oasc.SyncStrategy.syncToMe http://127.0.0.1:19958/collection1/ has no replicas
   [junit4]   2> 228494 T671 [control_collection] [shard1] [] []  oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:19958/collection1/ shard1
   [junit4]   2> 228494 T671 [control_collection] [shard1] [] []  oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 228495 T675 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 228495 T663 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 228496 T637 [] [] [] []  oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 228498 T663 [] [] [] []  oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 228498 T664 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:19958",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 228601 T675 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 228601 T663 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 228814 T671 [control_collection] [shard1] [] []  oasc.ZkController.register We are http://127.0.0.1:19958/collection1/ and leader is http://127.0.0.1:19958/collection1/
   [junit4]   2> 228815 T671 [control_collection] [shard1] [] []  oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:19958
   [junit4]   2> 228815 T671 [control_collection] [shard1] [] []  oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 228815 T671 [control_collection] [shard1] [] [collection1]  oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 228815 T671 [control_collection] [shard1] [] [collection1]  oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 228817 T663 [] [] [] []  oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 228817 T664 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:19958",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:19958_"} current state version: 3
   [junit4]   2> 228818 T664 [] [] [] []  oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:19958",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:19958_"}
   [junit4]   2> 228921 T663 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 228921 T675 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 229984 T637 [] [] [] []  oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/shard-1-001/cores/collection1
   [junit4]   2> 229987 T637 [] [] [] []  oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/shard-1-001
   [junit4]   2> 229987 T637 [] [] [] []  oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 229990 T637 [] [] [] []  oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:19964
   [junit4]   2> 229990 T637 [] [] [] []  oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/tempDir-001/jetty1, hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/shard-1-001/cores, solrconfig=solrconfig.xml, hostPort=19963}
   [junit4]   2> 229990 T637 [] [] [] []  oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 229991 T637 [] [] [] []  oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/shard-1-001/'
   [junit4]   2> 230020 T637 [] [] [] []  oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/shard-1-001/solr.xml
   [junit4]   2> 230055 T637 [] [] [] []  oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/shard-1-001/cores
   [junit4]   2> 230055 T637 [] [] [] []  oasc.CoreContainer.<init> New CoreContainer 952773271
   [junit4]   2> 230055 T637 [] [] [] []  oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/shard-1-001/]
   [junit4]   2> 230056 T637 [] [] [] []  oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/shard-1-001/lib
   [junit4]   2> 230056 T637 [] [] [] []  oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/shard-1-001/lib).
   [junit4]   2> 230065 T637 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 230065 T637 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 230066 T637 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 230066 T637 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 230066 T637 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 230066 T637 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 230066 T637 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 230067 T637 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 230067 T637 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 230067 T637 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 230067 T637 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 230068 T637 [] [] [] []  oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 230068 T637 [] [] [] []  oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 230068 T637 [] [] [] []  oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 230068 T637 [] [] [] []  oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 230069 T637 [] [] [] []  oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:19950/solr
   [junit4]   2> 230069 T637 [] [] [] []  oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 230069 T637 [] [] [] []  oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 230070 T637 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 230071 T687 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@38a52698 name:ZooKeeperConnection Watcher:127.0.0.1:19950 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 230072 T637 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 230072 T637 [] [] [] []  oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 230074 T637 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 230075 T690 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4fc9af86 name:ZooKeeperConnection Watcher:127.0.0.1:19950/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 230075 T637 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 230079 T637 [] [] [] []  oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 231082 T637 [] [] [] []  oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:19963_
   [junit4]   2> 231083 T637 [] [] [] []  oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:19963_
   [junit4]   2> 231086 T637 [] [] [] []  oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 231089 T637 [] [] [] []  oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/shard-1-001/cores
   [junit4]   2> 231091 T637 [] [] [] []  oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {shard=, loadOnStartup=true, coreNodeName=, dataDir=data/, config=solrconfig.xml, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/shard-1-001/cores/collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/shard-1-001/cores/collection1/, collection=collection1, schema=schema.xml, name=collection1}
   [junit4]   2> 231091 T637 [] [] [] []  oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/shard-1-001/cores/collection1/
   [junit4]   2> 231092 T637 [] [] [] []  oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 231093 T691 [collection1] [] [] [collection1]  oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 231093 T691 [collection1] [] [] [collection1]  oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 231094 T663 [] [] [] []  oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 231094 T691 [collection1] [] [] [collection1]  oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 231095 T664 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:19963",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:19963_"} current state version: 4
   [junit4]   2> 231095 T664 [] [] [] []  oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:19963",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:19963_"}
   [junit4]   2> 231095 T664 [] [] [] []  oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 231096 T664 [] [] [] []  oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 231097 T663 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 231097 T675 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 231097 T690 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 232094 T691 [collection1] [] [] [collection1]  oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 232095 T691 [collection1] [] [] [collection1]  oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 232095 T691 [collection1] [] [] [collection1]  oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 232096 T691 [collection1] [] [] [collection1]  oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 232096 T691 [collection1] [] [] [collection1]  oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 232096 T691 [collection1] [] [] [collection1]  oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/shard-1-001/cores/collection1/'
   [junit4]   2> 232113 T691 [collection1] [] [] [collection1]  oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 232116 T691 [collection1] [] [] [collection1]  oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 232126 T691 [collection1] [] [] [collection1]  oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 232148 T691 [collection1] [] [] [collection1]  oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 232197 T691 [collection1] [] [] [collection1]  oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 232298 T691 [collection1] [] [] [collection1]  oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 232309 T691 [collection1] [] [] [collection1]  oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 232697 T691 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 232714 T691 [collection1] [] [] [collection1]  oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 232717 T691 [collection1] [] [] [collection1]  oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 232728 T691 [collection1] [] [] [collection1]  oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 232733 T691 [collection1] [] [] [collection1]  oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 232737 T691 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 232737 T691 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 232738 T691 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 232738 T691 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 232738 T691 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 232739 T691 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 232739 T691 [collection1] [] [] [collection1]  oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 232739 T691 [collection1] [] [] [collection1]  oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 232740 T691 [collection1] [] [] [collection1]  oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 232740 T691 [collection1] [] [] [collection1]  oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3cd74bcf
   [junit4]   2> 232741 T691 [collection1] [] [] [collection1]  oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/shard-1-001/cores/collection1/data
   [junit4]   2> 232741 T691 [collection1] [] [] [collection1]  oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 232742 T691 [collection1] [] [] [collection1]  oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 232742 T691 [collection1] [] [] [collection1]  oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C1B43108DE3652A-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 232742 T691 [collection1] [] [] [collection1]  oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=45, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6125429853411988]
   [junit4]   2> 232743 T691 [collection1] [] [] [collection1]  oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@740512ef lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@791e446a),segFN=segments_1,generation=1}
   [junit4]   2> 232744 T691 [collection1] [] [] [collection1]  oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 232753 T691 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 232753 T691 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 232754 T691 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 232754 T691 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 232754 T691 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 232754 T691 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 232755 T691 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 232755 T691 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 232756 T691 [collection1] [] [] [collection1]  oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 232774 T691 [collection1] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 232777 T691 [collection1] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 232779 T691 [collection1] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 232782 T691 [collection1] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 232792 T691 [collection1] [] [] [collection1]  oasc.RequestHandlers.initHandlersFromConfig Registered paths: /config,/admin/logging,/admin/file,/admin/plugins,standard,/update,/admin/segments,/update/json/docs,/admin/threads,/admin/ping,/replication,/admin/system,/get,/admin/mbeans,/schema,/update/json,/admin/luke,/update/csv,/admin/properties
   [junit4]   2> 232794 T691 [collection1] [] [] [collection1]  oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 232796 T691 [collection1] [] [] [collection1]  oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 232796 T691 [collection1] [] [] [collection1]  oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 232797 T691 [collection1] [] [] [collection1]  oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 232798 T691 [collection1] [] [] [collection1]  oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 232799 T691 [collection1] [] [] [collection1]  oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=38, maxMergeAtOnceExplicit=27, maxMergedSegmentMB=73.0263671875, floorSegmentMB=1.0185546875, forceMergeDeletesPctAllowed=2.974923965248074, segmentsPerTier=47.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 232799 T691 [collection1] [] [] [collection1]  oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@740512ef lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@791e446a),segFN=segments_1,generation=1}
   [junit4]   2> 232800 T691 [collection1] [] [] [collection1]  oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 232800 T691 [collection1] [] [] [collection1]  oass.SolrIndexSearcher.<init> Opening Searcher@3fca49f9[collection1] main
   [junit4]   2> 232800 T691 [collection1] [] [] [collection1]  oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 232801 T691 [collection1] [] [] [collection1]  oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 232801 T691 [collection1] [] [] [collection1]  oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 232802 T691 [collection1] [] [] [collection1]  oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 232802 T691 [collection1] [] [] [collection1]  oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 232802 T691 [collection1] [] [] [collection1]  oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 232803 T691 [collection1] [] [] [collection1]  oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 232803 T691 [collection1] [] [] [collection1]  oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 232803 T691 [collection1] [] [] [collection1]  oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 232803 T691 [collection1] [] [] [collection1]  oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 232805 T692 [collection1] [] [] [collection1]  oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3fca49f9[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 232805 T691 [collection1] [] [] [collection1]  oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 232806 T695 [collection1] [shard1] [] []  oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:19963 collection:collection1 shard:shard1
   [junit4]   2> 232806 T637 [] [] [] []  oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 232806 T637 [] [] [] []  oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 232807 T695 [collection1] [shard1] [] []  oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 232810 T695 [collection1] [shard1] [] []  oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 232812 T663 [] [] [] []  oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 232812 T695 [collection1] [shard1] [] []  oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 232812 T695 [collection1] [shard1] [] []  oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C120 name=collection1 org.apache.solr.core.SolrCore@c47f3 url=http://127.0.0.1:19963/collection1 node=127.0.0.1:19963_ C120_STATE=coll:collection1 core:collection1 props:{core=collection1, state=down, base_url=http://127.0.0.1:19963, node_name=127.0.0.1:19963_}
   [junit4]   2> 232812 T695 [collection1] [shard1] [] []  C120 P19963 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:19963/collection1/
   [junit4]   2> 232812 T664 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 232812 T695 [collection1] [shard1] [] []  C120 P19963 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 232813 T695 [collection1] [shard1] [] []  C120 P19963 oasc.SyncStrategy.syncToMe http://127.0.0.1:19963/collection1/ has no replicas
   [junit4]   2> 232813 T695 [collection1] [shard1] [] []  oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:19963/collection1/ shard1
   [junit4]   2> 232813 T695 [collection1] [shard1] [] []  oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 232814 T690 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 232814 T675 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 232814 T663 [] [] [] []  oascc.ZkStateRead

[...truncated too long message...]

.0.0.1:15993 15993
   [junit4]   2> 1651090 T4513 [] [] [] []  oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1651090 T4512 [control_collection] [shard1] [] [collection1]  oasc.SocketProxy.close WARN Closing 6 connections to: http://127.0.0.1:16006/hk/ho
   [junit4]   2> 1651091 T4512 [control_collection] [shard1] [] [collection1]  oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:15998/hk/ho
   [junit4]   2> 1651091 T4512 [control_collection] [shard1] [] [collection1]  oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16012/hk/ho
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=7C1B43108DE3652A -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ms -Dtests.timezone=Africa/Dar_es_Salaam -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   70.8s J2 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:16006/hk/ho/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([7C1B43108DE3652A:F44F7CCA231F08D2]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:16006/hk/ho/collection1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 49 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
   [junit4]    > 	... 54 more
   [junit4]   2> 1651112 T4512 [control_collection] [shard1] [] [collection1]  oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 7C1B43108DE3652A-001
   [junit4]   2> 70756 T4511 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=FastDecompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=18664, maxDocsPerChunk=3, blockSize=5), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=18664, blockSize=5)), sim=DefaultSimilarity, locale=ms, timezone=Africa/Dar_es_Salaam
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=81604280,total=281018368
   [junit4]   2> NOTE: All tests run in this JVM: [TestLMDirichletSimilarityFactory, DocValuesTest, TestQuerySenderNoQuery, TestJmxIntegration, DirectSolrConnectionTest, FastVectorHighlighterTest, TestNRTOpen, FileUtilsTest, ZkNodePropsTest, TestLeaderElectionZkExpiry, TestPHPSerializedResponseWriter, TestFreeTextSuggestions, TestReplicationHandlerBackup, TimeZoneUtilsTest, TestIndexingPerformance, TestReloadAndDeleteDocs, OverseerTest, TestReplicationHandler, RollingRestartTest, TestDocSet, SignatureUpdateProcessorFactoryTest, TestPartialUpdateDeduplication, SSLMigrationTest, TestMiniSolrCloudClusterSSL, VMParamsZkACLAndCredentialsProvidersTest, ClusterStateUpdateTest, TestManagedSchemaDynamicFieldResource, XsltUpdateRequestHandlerTest, LeaderElectionIntegrationTest, TestReload, SuggestComponentTest, TestCharFilters, DateMathParserTest, TestConfig, DistributedFacetPivotWhiteBoxTest, CoreAdminCreateDiscoverTest, TestDistribDocBasedVersion, BlockCacheTest, TestStandardQParsers, TestCSVResponseWriter, SyncSliceTest, ConvertedLegacyTest, SolrTestCaseJ4Test, TestFileDictionaryLookup, HdfsChaosMonkeySafeLeaderTest, TestReversedWildcardFilterFactory, TestDistributedGrouping, TestQueryUtils, SpellPossibilityIteratorTest, TestCloudPivotFacet, SolrInfoMBeanTest, CacheHeaderTest, TestMergePolicyConfig, TestCopyFieldCollectionResource, TestDefaultSearchFieldResource, DeleteReplicaTest, BasicDistributedZk2Test, BasicDistributedZkTest, CustomCollectionTest, DistribDocExpirationUpdateProcessorTest, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J2 in 71.54s, 1 test, 1 error <<< FAILURES!

[...truncated 685 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 475 suites, 1894 tests, 3 errors, 62 ignored (22 assumptions)

Total time: 54 minutes 10 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 214578262 bytes
Compression is 0.0%
Took 40 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2828 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2828/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:11615/u_anl/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:11615/u_anl/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([CB95126E385C62EE:43C12DB496A00F16]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([CB95126E385C62EE:43C12DB496A00F16]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([CB95126E385C62EE:43C12DB496A00F16]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9997 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/init-core-data-001
   [junit4]   2> 1346411 T5511 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /u_anl/
   [junit4]   2> 1346416 T5511 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1346416 T5512 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1346516 T5511 oasc.ZkTestServer.run start zk server on port:11556
   [junit4]   2> 1346517 T5511 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1346517 T5511 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1346521 T5519 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@33222bb2 name:ZooKeeperConnection Watcher:127.0.0.1:11556 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1346521 T5511 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1346522 T5511 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1346522 T5511 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1346525 T5511 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1346526 T5511 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1346527 T5522 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2c8fc850 name:ZooKeeperConnection Watcher:127.0.0.1:11556/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1346527 T5511 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1346528 T5511 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1346528 T5511 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1346530 T5511 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1346531 T5511 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1346532 T5511 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1346534 T5511 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1346535 T5511 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1346537 T5511 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1346537 T5511 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1346640 T5511 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1346640 T5511 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1346642 T5511 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1346642 T5511 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1346643 T5511 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1346644 T5511 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1346645 T5511 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1346646 T5511 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1346647 T5511 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1346648 T5511 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1346649 T5511 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1346650 T5511 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1346651 T5511 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1346652 T5511 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1346653 T5511 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1346653 T5511 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1346655 T5511 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1346655 T5511 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1347676 T5511 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/control-001/cores/collection1
   [junit4]   2> 1347680 T5511 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1347683 T5511 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:11560
   [junit4]   2> 1347683 T5511 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=11559, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/tempDir-001/control/data, hostContext=/u_anl}
   [junit4]   2> 1347683 T5511 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 1347684 T5511 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/control-001/'
   [junit4]   2> 1347715 T5511 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/control-001/solr.xml
   [junit4]   2> 1347753 T5511 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/control-001/cores
   [junit4]   2> 1347753 T5511 oasc.CoreContainer.<init> New CoreContainer 1009090251
   [junit4]   2> 1347753 T5511 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/control-001/]
   [junit4]   2> 1347753 T5511 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/control-001/lib
   [junit4]   2> 1347754 T5511 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/control-001/lib).
   [junit4]   2> 1347763 T5511 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1347763 T5511 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1347763 T5511 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1347764 T5511 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1347764 T5511 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1347764 T5511 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1347764 T5511 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1347764 T5511 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1347765 T5511 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1347765 T5511 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1347765 T5511 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1347765 T5511 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1347766 T5511 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1347766 T5511 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1347766 T5511 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1347766 T5511 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:11556/solr
   [junit4]   2> 1347767 T5511 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1347767 T5511 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1347767 T5511 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1347769 T5534 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@108a6a01 name:ZooKeeperConnection Watcher:127.0.0.1:11556 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1347770 T5511 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1347770 T5511 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1347773 T5511 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1347774 T5537 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3b4e1afb name:ZooKeeperConnection Watcher:127.0.0.1:11556/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1347774 T5511 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1347776 T5511 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1347778 T5511 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1347780 T5511 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1347782 T5511 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1347784 T5511 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1347787 T5511 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1347788 T5511 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1347789 T5511 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1347790 T5511 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:11559_u_anl
   [junit4]   2> 1347791 T5511 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:11559_u_anl
   [junit4]   2> 1347793 T5511 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1347794 T5511 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1347795 T5511 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1347797 T5511 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:11559_u_anl
   [junit4]   2> 1347797 T5511 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1347798 T5511 oasc.Overseer.start Overseer (id=93534911433736195-127.0.0.1:11559_u_anl-n_0000000000) starting
   [junit4]   2> 1347800 T5511 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1347805 T5511 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 1347806 T5539 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1347806 T5511 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1347806 T5538 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1347810 T5511 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/control-001/cores
   [junit4]   2> 1347813 T5511 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {shard=, dataDir=data/, schema=schema.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/control-001/cores/collection1/, coreNodeName=, transient=false, config=solrconfig.xml, name=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/control-001/cores/collection1, collection=control_collection, loadOnStartup=true}
   [junit4]   2> 1347813 T5511 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/control-001/cores/collection1/
   [junit4]   2> 1347813 T5511 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1347814 T5541 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1347814 T5541 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1347815 T5537 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1347815 T5541 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1347816 T5538 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:11559_u_anl",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11559/u_anl"} current state version: 0
   [junit4]   2> 1347816 T5538 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:11559_u_anl",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11559/u_anl"}
   [junit4]   2> 1347817 T5538 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 1347817 T5538 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1347818 T5537 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1348816 T5541 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1348817 T5541 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1348817 T5541 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1348818 T5541 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1348818 T5541 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1348818 T5541 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/control-001/cores/collection1/'
   [junit4]   2> 1348836 T5541 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1348838 T5541 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1348848 T5541 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1348871 T5541 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 1348921 T5541 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1349022 T5541 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1349032 T5541 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1349434 T5541 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1349449 T5541 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1349452 T5541 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1349460 T5541 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1349464 T5541 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1349468 T5541 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1349469 T5541 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1349469 T5541 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1349469 T5541 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1349470 T5541 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1349470 T5541 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1349471 T5541 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1349471 T5541 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1349471 T5541 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/control-001/cores/collection1/data/
   [junit4]   2> 1349471 T5541 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@11b9e4ca
   [junit4]   2> 1349472 T5541 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/control-001/cores/collection1/data
   [junit4]   2> 1349473 T5541 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/control-001/cores/collection1/data/index/
   [junit4]   2> 1349473 T5541 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1349473 T5541 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/control-001/cores/collection1/data/index
   [junit4]   2> 1349474 T5541 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=22, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=13.947265625, floorSegmentMB=1.5234375, forceMergeDeletesPctAllowed=17.239313209902956, segmentsPerTier=32.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1349474 T5541 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5a55d396 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@38b64b2f),segFN=segments_1,generation=1}
   [junit4]   2> 1349475 T5541 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1349480 T5541 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1349481 T5541 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1349481 T5541 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1349481 T5541 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1349481 T5541 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1349482 T5541 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1349482 T5541 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1349482 T5541 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1349483 T5541 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1349502 T5541 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1349504 T5541 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1349506 T5541 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1349508 T5541 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1349513 T5541 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/logging,/replication,/admin/system,/admin/segments,/update/json,/admin/threads,/update/csv,/admin/file,/schema,/get,/admin/mbeans,standard,/config,/admin/ping,/admin/luke,/update,/update/json/docs,/admin/properties,/admin/plugins
   [junit4]   2> 1349515 T5541 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1349517 T5541 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1349517 T5541 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1349519 T5541 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1349519 T5541 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1349520 T5541 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=535510694, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1349520 T5541 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5a55d396 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@38b64b2f),segFN=segments_1,generation=1}
   [junit4]   2> 1349520 T5541 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1349521 T5541 oass.SolrIndexSearcher.<init> Opening Searcher@30bcfb68[collection1] main
   [junit4]   2> 1349521 T5541 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1349522 T5541 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1349522 T5541 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1349523 T5541 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1349523 T5541 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1349523 T5541 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1349524 T5541 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1349524 T5541 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1349524 T5541 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1349524 T5541 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1349525 T5541 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1349525 T5542 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@30bcfb68[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1349526 T5545 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:11559/u_anl collection:control_collection shard:shard1
   [junit4]   2> 1349526 T5511 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 1349526 T5511 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1349526 T5545 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1349529 T5511 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1349529 T5511 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1349530 T5545 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1349531 T5549 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@138665ea name:ZooKeeperConnection Watcher:127.0.0.1:11556/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1349531 T5511 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1349531 T5511 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1349531 T5511 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1349531 T5537 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1349533 T5545 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1349533 T5545 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C3986 name=collection1 org.apache.solr.core.SolrCore@3c58ad4f url=http://127.0.0.1:11559/u_anl/collection1 node=127.0.0.1:11559_u_anl C3986_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:11559_u_anl, state=down, core=collection1, base_url=http://127.0.0.1:11559/u_anl}
   [junit4]   2> 1349533 T5545 C3986 P11559 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:11559/u_anl/collection1/
   [junit4]   2> 1349533 T5538 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 1349533 T5545 C3986 P11559 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1349534 T5545 C3986 P11559 oasc.SyncStrategy.syncToMe http://127.0.0.1:11559/u_anl/collection1/ has no replicas
   [junit4]   2> 1349534 T5545 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:11559/u_anl/collection1/ shard1
   [junit4]   2> 1349534 T5545 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1349534 T5511 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1349535 T5549 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1349535 T5511 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 1349535 T5537 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1349535 T5511 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1349536 T5511 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1349537 T5552 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3154f6b4 name:ZooKeeperConnection Watcher:127.0.0.1:11556/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1349538 T5511 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1349538 T5511 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1349539 T5537 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1349540 T5538 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11559/u_anl",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 1349542 T5538 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 1349543 T5538 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 1349543 T5537 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1349543 T5549 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1349546 T5537 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1349546 T5538 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 1349547 T5549 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1349547 T5537 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1349590 T5545 oasc.ZkController.register We are http://127.0.0.1:11559/u_anl/collection1/ and leader is http://127.0.0.1:11559/u_anl/collection1/
   [junit4]   2> 1349591 T5545 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:11559/u_anl
   [junit4]   2> 1349591 T5545 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1349591 T5545 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1349591 T5545 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1349592 T5537 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1349593 T5538 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:11559_u_anl",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11559/u_anl",
   [junit4]   2> 	  "core_node_name":"core_node1"} current state version: 4
   [junit4]   2> 1349594 T5538 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:11559_u_anl",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11559/u_anl",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 1349697 T5537 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1349697 T5549 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1350575 T5511 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/shard-1-001/cores/collection1
   [junit4]   2> 1350577 T5511 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/shard-1-001
   [junit4]   2> 1350578 T5511 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1350580 T5511 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:11573
   [junit4]   2> 1350581 T5511 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/shard-1-001/cores, hostPort=11572, solrconfig=solrconfig.xml, hostContext=/u_anl, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/tempDir-001/jetty1}
   [junit4]   2> 1350581 T5511 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 1350581 T5511 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/shard-1-001/'
   [junit4]   2> 1350615 T5511 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/shard-1-001/solr.xml
   [junit4]   2> 1350655 T5511 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/shard-1-001/cores
   [junit4]   2> 1350656 T5511 oasc.CoreContainer.<init> New CoreContainer 1636496419
   [junit4]   2> 1350656 T5511 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/shard-1-001/]
   [junit4]   2> 1350656 T5511 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/shard-1-001/lib
   [junit4]   2> 1350657 T5511 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/shard-1-001/lib).
   [junit4]   2> 1350666 T5511 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1350667 T5511 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1350667 T5511 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1350667 T5511 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1350667 T5511 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1350667 T5511 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1350668 T5511 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1350668 T5511 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1350668 T5511 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1350668 T5511 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1350669 T5511 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1350669 T5511 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1350669 T5511 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1350670 T5511 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1350670 T5511 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1350670 T5511 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:11556/solr
   [junit4]   2> 1350670 T5511 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1350670 T5511 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1350671 T5511 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1350673 T5564 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1423796e name:ZooKeeperConnection Watcher:127.0.0.1:11556 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1350674 T5511 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1350674 T5511 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1350676 T5511 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1350677 T5567 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@609fae61 name:ZooKeeperConnection Watcher:127.0.0.1:11556/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1350677 T5511 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1350681 T5511 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1351685 T5511 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:11572_u_anl
   [junit4]   2> 1351686 T5511 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:11572_u_anl
   [junit4]   2> 1351689 T5511 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1351691 T5511 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/shard-1-001/cores
   [junit4]   2> 1351693 T5511 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/shard-1-001/cores/collection1/, dataDir=data/, coreNodeName=, shard=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/shard-1-001/cores/collection1, name=collection1, collection=collection1, transient=false, config=solrconfig.xml, schema=schema.xml, loadOnStartup=true}
   [junit4]   2> 1351693 T5511 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/shard-1-001/cores/collection1/
   [junit4]   2> 1351693 T5511 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1351695 T5568 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1351696 T5568 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1351697 T5537 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1351697 T5568 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 1351697 T5568 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 1351698 T5538 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:11572_u_anl",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11572/u_anl"} current state version: 5
   [junit4]   2> 1351698 T5568 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 1351698 T5538 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:11572_u_anl",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11572/u_anl"}
   [junit4]   2> 1351698 T5538 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 1351698 T5538 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1351699 T5568 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 1351699 T5568 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1351699 T5538 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 1351700 T5567 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1351700 T5567 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 1352699 T5568 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1352700 T5568 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1352700 T5568 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1352701 T5568 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1352701 T5568 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1352701 T5568 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/shard-1-001/cores/collection1/'
   [junit4]   2> 1352719 T5568 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1352721 T5568 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1352731 T5568 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1352753 T5568 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 1352801 T5568 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1352902 T5568 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1352909 T5568 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1353296 T5568 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1353311 T5568 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1353314 T5568 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1353323 T5568 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1353327 T5568 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1353331 T5568 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1353332 T5568 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1353332 T5568 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1353332 T5568 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1353333 T5568 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1353333 T5568 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1353333 T5568 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1353334 T5568 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1353334 T5568 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 1353334 T5568 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@11b9e4ca
   [junit4]   2> 1353335 T5568 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/shard-1-001/cores/collection1/data
   [junit4]   2> 1353335 T5568 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 1353336 T5568 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1353336 T5568 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 1353336 T5568 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=22, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=13.947265625, floorSegmentMB=1.5234375, forceMergeDeletesPctAllowed=17.239313209902956, segmentsPerTier=32.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1353337 T5568 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@562d214c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2ce530c7),segFN=segments_1,generation=1}
   [junit4]   2> 1353337 T5568 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1353344 T5568 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1353345 T5568 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1353345 T5568 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1353345 T5568 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1353345 T5568 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1353346 T5568 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1353346 T5568 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1353346 T5568 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1353347 T5568 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1353366 T5568 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1353368 T5568 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1353370 T5568 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1353381 T5568 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1353386 T5568 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/logging,/replication,/admin/system,/admin/segments,/update/json,/admin/threads,/update/csv,/admin/file,/schema,/get,/admin/mbeans,standard,/config,/admin/ping,/admin/luke,/update,/update/json/docs,/admin/properties,/admin/plugins
   [junit4]   2> 1353388 T5568 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1353389 T5568 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1353389 T5568 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1353390 T5568 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1353390 T5568 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1353392 T5568 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=535510694, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1353392 T5568 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@562d214c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2ce530c7),segFN=segments_1,generation=1}
   [junit4]   2> 1353392 T5568 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1353393 T5568 oass.SolrIndexSearcher.<init> Opening Searcher@3030ac0c[collection1] main
   [junit4]   2> 1353393 T5568 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1353393 T5568 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1353394 T5568 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1353394 T5568 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1353394 T5568 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1353394 T5568 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1353395 T5568 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1353395 T5568 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1353395 T5568 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1353395 T5568 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1353396 T5569 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3030ac0c[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1353397 T5568 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1353398 T5572 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:11572/u_anl collection:collection1 shard:shard1
   [junit4]   2> 1353398 T5511 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 1353398 T5511 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1353398 T5572 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1353402 T5572 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1353404 T5537 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1353404 T5572 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1353404 T5572 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C3987 name=collection1 org.apache.solr.core.SolrCore@4f3d7801 url=http://127.0.0.1:11572/u_anl/collection1 node=127.0.0.1:11572_u_anl C3987_STATE=coll:collection1 core:collection1 props:{node_name=127.0.0.1:11572_u_anl, state=down, core=collection1, base_url=http://127.0.0.1:11572/u_anl}
   [junit4]   2> 1353405 T5572 C3987 P11572 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:11572/u_anl/collection1/
   [junit4]   2> 1353405 T5538 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 1353405 T5572 C3987 P11572 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1353406 T5572 C3987 P11572 oasc.SyncStrategy.syncToMe http://127.0.0.1:11572/u_anl/collection1/ has no replicas
   [junit4]   2> 1353406 T5572 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:11572/u_anl/collection1/ shard1
   [junit4]   2> 1353406 T5572 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1353406 T5538 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 1353407 T5567 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1353408 T5567 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 1353410 T5538 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11572/u_anl",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 1353415 T5538 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 1353415 T5567 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1353416 T5567 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 1353417 T5537 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1353865 T5572 oasc.ZkController.register We are http://127.0.0.1:11572/u_anl/collection1/ and leader is http://127.0.0.1:11572/u_anl/collection1/
   [junit4]   2> 1353866 T5572 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:11572/u_anl
   [junit4]   2> 1353866 T5572 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1353866 T5572 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 1353866 T5572 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1353867 T5537 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1353868 T5538 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:11572_u_anl",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11572/u_anl",
   [junit4]   2> 	  "core_node_name":"core_node1"} current state version: 5
   [junit4]   2> 1353869 T5538 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:11572_u_anl",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:11572/u_anl",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 1353870 T5572 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 1353870 T5538 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 3
   [junit4]   2> 1353870 T5567 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1353871 T5567 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 4 
   [junit4]   2> 1354648 T5511 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/shard-2-001/cores/collection1
   [junit4]   2> 1354650 T5511 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/shard-2-001
   [junit4]   2> 1354651 T5511 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1354653 T5511 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:11579
   [junit4]   2> 1354653 T5511 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=11578, hostContext=/u_anl, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/shard-2-001/cores, solrconfig=solrconfig.xml, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/tempDir-001/jetty2}
   [junit4]   2> 1354654 T5511 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 1354654 T5511 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/shard-2-001/'
   [junit4]   2> 1354686 T5511 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/shard-2-001/solr.xml
   [junit4]   2> 1354723 T5511 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/shard-2-001/cores
   [junit4]   2> 1354723 T5511 oasc.CoreContainer.<init> New CoreContainer 1891715223
   [junit4]   2> 1354724 T5511 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest CB95126E385C62EE-001/shard-2-001

[...truncated too long message...]

llection1/data
   [junit4]   2> 1532096 T3139 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/_dxw/l,null}
   [junit4]   2> 1532250 T3139 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:11985 11985
   [junit4]   2> 1532316 T3140 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:11985 11985
   [junit4]   2> 1532317 T3140 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 		2	/solr/collections/collection1/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1532317 T3139 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:12016/_dxw/l
   [junit4]   2> 1532317 T3139 oasc.SocketProxy.close WARN Closing 6 connections to: http://127.0.0.1:12006/_dxw/l
   [junit4]   2> 1532318 T3139 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:11992/_dxw/l
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=CB95126E385C62EE -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=uk_UA -Dtests.timezone=Asia/Kamchatka -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   67.7s J0 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([CB95126E385C62EE:43C12DB496A00F16]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 1532330 T3139 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest CB95126E385C62EE-001
   [junit4]   2> 67734 T3138 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {text=FST50, id=PostingsFormat(name=Direct), range_facet_l=PostingsFormat(name=LuceneFixedGap), timestamp=PostingsFormat(name=LuceneFixedGap), _version_=PostingsFormat(name=LuceneFixedGap), intDefault=PostingsFormat(name=LuceneFixedGap), a_t=PostingsFormat(name=LuceneFixedGap), multiDefault=PostingsFormat(name=Memory doPackFST= false)}, docValues:{timestamp=DocValuesFormat(name=Memory)}, sim=DefaultSimilarity, locale=uk_UA, timezone=Asia/Kamchatka
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=88947184,total=294649856
   [junit4]   2> NOTE: All tests run in this JVM: [RecoveryZkTest, ZkCLITest, TestDynamicFieldResource, SpellingQueryConverterTest, SolrCoreCheckLockOnStartupTest, StatelessScriptUpdateProcessorFactoryTest, TestRecovery, DistanceUnitsTest, TestReRankQParserPlugin, DateRangeFieldTest, HttpPartitionTest, TestManagedResourceStorage, TriLevelCompositeIdRoutingTest, TestJsonFacets, DirectUpdateHandlerTest, TestManagedSynonymFilterFactory, TestTolerantSearch, TestFastWriter, TestSurroundQueryParser, TestSchemaNameResource, TestJettySolrRunner, TestExactSharedStatsCache, TestFuzzyAnalyzedSuggestions, TestMissingGroups, AnalysisAfterCoreReloadTest, TestFaceting, UUIDFieldTest, ConvertedLegacyTest, TestShortCircuitedRequests, SampleTest, SharedFSAutoReplicaFailoverUtilsTest, TestCSVResponseWriter, TestSolrDynamicMBean, TestIBSimilarityFactory, SSLMigrationTest, TestSolrConfigHandlerCloud, TestSolrConfigHandler, SolrIndexSplitterTest, ZkSolrClientTest, SearchHandlerTest, TestDocBasedVersionConstraints, AddBlockUpdateTest, TestAnalyzeInfixSuggestions, TestDocSet, SuggestComponentTest, PreAnalyzedFieldTest, TestImplicitCoreProperties, TestStressRecovery, TestElisionMultitermQuery, TestSolrQueryParser, SimpleMLTQParserTest, DateMathParserTest, ParsingFieldUpdateProcessorsTest, UpdateParamsTest, AnalyticsMergeStrategyTest, ShardRoutingTest, TestCoreDiscovery, UnloadDistributedZkTest, DistribDocExpirationUpdateProcessorTest, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J0 in 68.27s, 1 test, 1 error <<< FAILURES!

[...truncated 688 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 475 suites, 1894 tests, 3 errors, 62 ignored (22 assumptions)

Total time: 52 minutes 49 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 206714404 bytes
Compression is 0.0%
Took 39 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2827 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2827/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:23638/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:23638/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([7C935111F5E670FD:F4C76ECB5B1A1D05]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:65391/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:65391/collection1
	at __randomizedtesting.SeedInfo.seed([7C935111F5E670FD:F4C76ECB5B1A1D05]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:65391/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37264/repfacttest_c8n_1x3_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37264/repfacttest_c8n_1x3_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([7C935111F5E670FD:F4C76ECB5B1A1D05]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37264/repfacttest_c8n_1x3_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 53 more




Build Log:
[...truncated 9708 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/init-core-data-001
   [junit4]   2> 347259 T1378 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 347263 T1378 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 347264 T1379 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 347363 T1378 oasc.ZkTestServer.run start zk server on port:37241
   [junit4]   2> 347364 T1378 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 347365 T1378 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 347370 T1386 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@401db8e0 name:ZooKeeperConnection Watcher:127.0.0.1:37241 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 347370 T1378 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 347371 T1378 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 347371 T1378 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 347374 T1378 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 347375 T1378 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 347377 T1389 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4ae881fb name:ZooKeeperConnection Watcher:127.0.0.1:37241/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 347377 T1378 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 347378 T1378 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 347378 T1378 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 347381 T1378 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 347383 T1378 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 347385 T1378 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 347388 T1378 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 347388 T1378 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 347395 T1378 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 347396 T1378 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 347500 T1378 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 347501 T1378 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 347504 T1378 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 347505 T1378 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 347508 T1378 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 347509 T1378 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 347512 T1378 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 347513 T1378 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 347519 T1378 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 347521 T1378 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 347524 T1378 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 347525 T1378 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 347528 T1378 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 347529 T1378 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 347532 T1378 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 347532 T1378 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 347535 T1378 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 347536 T1378 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 349293 T1378 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/control-001/cores/collection1
   [junit4]   2> 349297 T1378 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 349300 T1378 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:37247
   [junit4]   2> 349300 T1378 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=37246, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/tempDir-001/control/data, hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/control-001/cores}
   [junit4]   2> 349300 T1378 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 349301 T1378 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/control-001/'
   [junit4]   2> 349340 T1378 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/control-001/solr.xml
   [junit4]   2> 349389 T1378 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/control-001/cores
   [junit4]   2> 349389 T1378 oasc.CoreContainer.<init> New CoreContainer 1466264617
   [junit4]   2> 349390 T1378 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/control-001/]
   [junit4]   2> 349390 T1378 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/control-001/lib
   [junit4]   2> 349391 T1378 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/control-001/lib).
   [junit4]   2> 349401 T1378 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 349401 T1378 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 349401 T1378 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 349402 T1378 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 349402 T1378 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 349402 T1378 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 349402 T1378 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 349403 T1378 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 349403 T1378 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 349403 T1378 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 349403 T1378 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 349404 T1378 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 349404 T1378 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 349404 T1378 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 349405 T1378 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 349405 T1378 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:37241/solr
   [junit4]   2> 349405 T1378 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 349406 T1378 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 349407 T1378 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 349408 T1401 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2a68e82 name:ZooKeeperConnection Watcher:127.0.0.1:37241 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 349409 T1378 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 349410 T1378 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 349412 T1378 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 349413 T1404 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2a50e28b name:ZooKeeperConnection Watcher:127.0.0.1:37241/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 349414 T1378 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 349415 T1378 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 349418 T1378 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 349420 T1378 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 349423 T1378 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 349425 T1378 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 349428 T1378 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 349430 T1378 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 349431 T1378 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 349433 T1378 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37246_
   [junit4]   2> 349433 T1378 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:37246_
   [junit4]   2> 349435 T1378 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 349437 T1378 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 349439 T1378 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 349441 T1378 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:37246_
   [junit4]   2> 349441 T1378 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 349443 T1378 oasc.Overseer.start Overseer (id=93534054018056195-127.0.0.1:37246_-n_0000000000) starting
   [junit4]   2> 349445 T1378 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 349452 T1378 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 349453 T1406 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 349453 T1378 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 349454 T1405 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 349458 T1378 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/control-001/cores
   [junit4]   2> 349460 T1378 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {shard=, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/control-001/cores/collection1, loadOnStartup=true, coreNodeName=, collection=control_collection, name=collection1, config=solrconfig.xml, schema=schema.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/control-001/cores/collection1/, dataDir=data/}
   [junit4]   2> 349461 T1378 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/control-001/cores/collection1/
   [junit4]   2> 349461 T1378 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 349462 T1408 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 349462 T1408 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 349463 T1404 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 349463 T1408 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 349464 T1405 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:37246_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37246",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 0
   [junit4]   2> 349465 T1405 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:37246_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37246",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection"}
   [junit4]   2> 349465 T1405 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 349465 T1405 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 349467 T1404 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 350464 T1408 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 350466 T1408 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 350468 T1408 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 350469 T1408 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 350470 T1408 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 350470 T1408 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/control-001/cores/collection1/'
   [junit4]   2> 350487 T1408 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 350491 T1408 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 350506 T1408 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 350531 T1408 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 350603 T1408 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 350708 T1408 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 350724 T1408 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 351522 T1408 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 351544 T1408 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 351547 T1408 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 351615 T1408 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 351622 T1408 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 351630 T1408 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 351632 T1408 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 351632 T1408 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 351632 T1408 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 351634 T1408 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 351634 T1408 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 351635 T1408 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 351635 T1408 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 351636 T1408 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/control-001/cores/collection1/data/
   [junit4]   2> 351636 T1408 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2bc5e5b2
   [junit4]   2> 351637 T1408 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/control-001/cores/collection1/data
   [junit4]   2> 351638 T1408 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/control-001/cores/collection1/data/index/
   [junit4]   2> 351638 T1408 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 351638 T1408 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/control-001/cores/collection1/data/index
   [junit4]   2> 351639 T1408 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=20, maxMergeAtOnceExplicit=48, maxMergedSegmentMB=4.4580078125, floorSegmentMB=1.484375, forceMergeDeletesPctAllowed=12.50581052477761, segmentsPerTier=49.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 351640 T1408 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6c6112a7 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@590fc987),segFN=segments_1,generation=1}
   [junit4]   2> 351641 T1408 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 351648 T1408 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 351649 T1408 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 351649 T1408 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 351650 T1408 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 351650 T1408 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 351650 T1408 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 351651 T1408 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 351651 T1408 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 351651 T1408 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 351679 T1408 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 351682 T1408 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 351686 T1408 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 351689 T1408 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 351696 T1408 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/mbeans,/admin/logging,/config,/admin/system,/update,/admin/plugins,/admin/file,/schema,/update/json/docs,/admin/properties,/admin/luke,/replication,/admin/threads,/update/csv,/update/json,/admin/ping,/admin/segments,standard,/get
   [junit4]   2> 351698 T1408 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 351699 T1408 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 351700 T1408 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 351701 T1408 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 351702 T1408 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 351703 T1408 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=31, maxMergeAtOnceExplicit=32, maxMergedSegmentMB=9.0244140625, floorSegmentMB=0.3779296875, forceMergeDeletesPctAllowed=13.251938737068334, segmentsPerTier=21.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.2609325220910932
   [junit4]   2> 351704 T1408 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6c6112a7 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@590fc987),segFN=segments_1,generation=1}
   [junit4]   2> 351704 T1408 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 351705 T1408 oass.SolrIndexSearcher.<init> Opening Searcher@560c48bd[collection1] main
   [junit4]   2> 351705 T1408 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 351707 T1408 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 351707 T1408 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 351708 T1408 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 351708 T1408 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 351708 T1408 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 351709 T1408 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 351710 T1408 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 351710 T1408 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 351710 T1408 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 351711 T1409 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@560c48bd[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 351712 T1408 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 351715 T1412 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:37246 collection:control_collection shard:shard1
   [junit4]   2> 351716 T1378 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 351717 T1378 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 351717 T1412 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 351721 T1378 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 351722 T1378 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 351728 T1412 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 351732 T1404 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 351733 T1412 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 351734 T1405 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 351734 T1412 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C121 name=collection1 org.apache.solr.core.SolrCore@13a32288 url=http://127.0.0.1:37246/collection1 node=127.0.0.1:37246_ C121_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:37246_, core=collection1, base_url=http://127.0.0.1:37246, state=down}
   [junit4]   2> 351735 T1412 C121 P37246 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:37246/collection1/
   [junit4]   2> 351735 T1412 C121 P37246 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 351736 T1412 C121 P37246 oasc.SyncStrategy.syncToMe http://127.0.0.1:37246/collection1/ has no replicas
   [junit4]   2> 351736 T1412 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:37246/collection1/ shard1
   [junit4]   2> 351736 T1404 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 351736 T1412 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 351742 T1416 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@76471b44 name:ZooKeeperConnection Watcher:127.0.0.1:37241/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 351742 T1378 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 351744 T1378 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 351744 T1378 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 351748 T1404 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 351749 T1378 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 351750 T1405 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37246",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 351856 T1416 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 351856 T1404 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 351902 T1412 oasc.ZkController.register We are http://127.0.0.1:37246/collection1/ and leader is http://127.0.0.1:37246/collection1/
   [junit4]   2> 351903 T1412 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:37246
   [junit4]   2> 351903 T1412 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 351904 T1412 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 351904 T1412 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 351906 T1404 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 351908 T1405 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:37246_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37246",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 3
   [junit4]   2> 351909 T1405 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:37246_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37246",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection"}
   [junit4]   2> 352019 T1416 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 352019 T1404 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 353713 T1378 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/shard-1-001/cores/collection1
   [junit4]   2> 353717 T1378 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/shard-1-001
   [junit4]   2> 353719 T1378 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 353722 T1378 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:37254
   [junit4]   2> 353722 T1378 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/tempDir-001/jetty1, hostPort=37253, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/shard-1-001/cores, solrconfig=solrconfig.xml, hostContext=/}
   [junit4]   2> 353723 T1378 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 353724 T1378 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/shard-1-001/'
   [junit4]   2> 353787 T1378 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/shard-1-001/solr.xml
   [junit4]   2> 353888 T1378 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/shard-1-001/cores
   [junit4]   2> 353889 T1378 oasc.CoreContainer.<init> New CoreContainer 413301
   [junit4]   2> 353889 T1378 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/shard-1-001/]
   [junit4]   2> 353890 T1378 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/shard-1-001/lib
   [junit4]   2> 353891 T1378 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/shard-1-001/lib).
   [junit4]   2> 353911 T1378 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 353912 T1378 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 353913 T1378 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 353913 T1378 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 353914 T1378 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 353915 T1378 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 353915 T1378 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 353916 T1378 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 353916 T1378 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 353917 T1378 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 353918 T1378 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 353919 T1378 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 353920 T1378 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 353920 T1378 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 353921 T1378 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 353921 T1378 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:37241/solr
   [junit4]   2> 353922 T1378 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 353922 T1378 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 353923 T1378 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 353926 T1428 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@12f538ae name:ZooKeeperConnection Watcher:127.0.0.1:37241 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 353927 T1378 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 353929 T1378 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 353932 T1378 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 353935 T1431 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6890dad9 name:ZooKeeperConnection Watcher:127.0.0.1:37241/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 353936 T1378 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 353976 T1378 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 354985 T1378 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37253_
   [junit4]   2> 354987 T1378 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:37253_
   [junit4]   2> 354993 T1378 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 354998 T1378 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/shard-1-001/cores
   [junit4]   2> 355000 T1378 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, name=collection1, dataDir=data/, coreNodeName=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/shard-1-001/cores/collection1/, shard=, transient=false, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/shard-1-001/cores/collection1, loadOnStartup=true, collection=collection1}
   [junit4]   2> 355001 T1378 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/shard-1-001/cores/collection1/
   [junit4]   2> 355001 T1378 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 355003 T1432 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 355003 T1432 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 355005 T1404 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 355005 T1432 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 355006 T1405 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:37253_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37253",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1"} current state version: 4
   [junit4]   2> 355008 T1405 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:37253_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37253",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1"}
   [junit4]   2> 355008 T1405 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 355009 T1405 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 355011 T1404 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 355011 T1431 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 355011 T1416 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 356005 T1432 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 356006 T1432 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 356007 T1432 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 356007 T1432 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 356008 T1432 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 356009 T1432 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/shard-1-001/cores/collection1/'
   [junit4]   2> 356027 T1432 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 356032 T1432 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 356047 T1432 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 356074 T1432 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 356150 T1432 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 356253 T1432 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 356265 T1432 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 356734 T1432 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 356754 T1432 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 356757 T1432 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 356779 T1432 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 356785 T1432 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 356791 T1432 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 356793 T1432 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 356793 T1432 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 356794 T1432 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 356795 T1432 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 356796 T1432 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 356796 T1432 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 356796 T1432 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 356797 T1432 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 356797 T1432 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2bc5e5b2
   [junit4]   2> 356798 T1432 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/shard-1-001/cores/collection1/data
   [junit4]   2> 356799 T1432 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 356799 T1432 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 356799 T1432 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 356800 T1432 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=20, maxMergeAtOnceExplicit=48, maxMergedSegmentMB=4.4580078125, floorSegmentMB=1.484375, forceMergeDeletesPctAllowed=12.50581052477761, segmentsPerTier=49.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 356801 T1432 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@297c2093 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@143ea82c),segFN=segments_1,generation=1}
   [junit4]   2> 356801 T1432 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 356812 T1432 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 356812 T1432 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 356813 T1432 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 356813 T1432 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 356813 T1432 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 356814 T1432 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 356815 T1432 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 356815 T1432 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 356815 T1432 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 356840 T1432 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 356843 T1432 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 356846 T1432 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 356849 T1432 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 356858 T1432 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/mbeans,/admin/logging,/config,/admin/system,/update,/admin/plugins,/admin/file,/schema,/update/json/docs,/admin/properties,/admin/luke,/replication,/admin/threads,/update/csv,/update/json,/admin/ping,/admin/segments,standard,/get
   [junit4]   2> 356861 T1432 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 356863 T1432 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 356864 T1432 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 356866 T1432 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 356866 T1432 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 356867 T1432 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=31, maxMergeAtOnceExplicit=32, maxMergedSegmentMB=9.0244140625, floorSegmentMB=0.3779296875, forceMergeDeletesPctAllowed=13.251938737068334, segmentsPerTier=21.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.2609325220910932
   [junit4]   2> 356868 T1432 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@297c2093 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@143ea82c),segFN=segments_1,generation=1}
   [junit4]   2> 356868 T1432 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 356869 T1432 oass.SolrIndexSearcher.<init> Opening Searcher@316c77d7[collection1] main
   [junit4]   2> 356869 T1432 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 356871 T1432 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 356872 T1432 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 356872 T1432 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 356873 T1432 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 356873 T1432 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 356874 T1432 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 356874 T1432 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 356875 T1432 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 356875 T1432 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 356877 T1433 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@316c77d7[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 356878 T1432 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 356879 T1436 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:37253 collection:collection1 shard:shard1
   [junit4]   2> 356880 T1378 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 356881 T1378 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 356881 T1436 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 356890 T1436 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 356893 T1404 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 356896 T1405 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 356899 T1436 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 356899 T1431 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 356900 T1436 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 356899 T1404 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 356899 T1416 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> ASYNC  NEW_CORE C122 name=collection1 org.apache.solr.core.SolrCore@56da2f67 url=http://127.0.0.1:37253/collection1 node=127.0.0.1:37253_ C122_STATE=coll:collection1 core:collection1 props:{node_name=127.0.0.1:37253_, core=collection1, base_url=http://127.0.0.1:37253, state=down}
   [junit4]   2> 356901 T1436 C122 P37253 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:37253/collection1/
   [junit4]   2> 356902 T1436 C122 P37253 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 356902 T1436 C122 P37253 oasc.SyncStrategy.syncToMe http://127.0.0.1:37253/collection1/ has no replicas
   [junit4]   2> 356903 T1436 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:37253/collection1/ shard1
   [junit4]   2> 356903 T1436 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 356913 T1404 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 356915 T1405 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37253",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 357022 T1416 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 357022 T1431 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 357022 T1404 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 357070 T1436 oasc.ZkController.register We are http://127.0.0.1:37253/collection1/ and leader is http://127.0.0.1:37253/collection1/
   [junit4]   2> 357071 T1436 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:37253
   [junit4]   2> 357071 T1436 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 357072 T1436 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 357072 T1436 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 357075 T1404 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 357076 T1405 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:37253_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37253",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1"} current state version: 7
   [junit4]   2> 357078 T1405 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:37253_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37253",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1"}
   [junit4]   2> 357185 T1404 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 357185 T1416 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 357185 T1431 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 358796 T1378 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/shard-2-001/cores/collection1
   [junit4]   2> 358799 T1378 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/shard-2-001
   [junit4]   2> 358800 T1378 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 358804 T1378 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:37260
   [junit4]   2> 358805 T1378 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/shard-2-001/cores, hostContext=/, hostPort=37259, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/tempDir-001/jetty2, solrconfig=solrconfig.xml}
   [junit4]   2> 358806 T1378 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 358806 T1378 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/shard-2-001/'
   [junit4]   2> 358862 T1378 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/shard-2-001/solr.xml
   [junit4]   2> 358931 T1378 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/shard-2-001/cores
   [junit4]   2> 358932 T1378 oasc.CoreContainer.<init> New CoreContainer 1782261825
   [junit4]   2> 358932 T1378 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/shard-2-001/]
   [junit4]   2> 358933 T1378 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/shard-2-001/lib
   [junit4]   2> 358934 T1378 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 7C935111F5E670FD-001/shard-2-001/lib).
   [junit4]   2> 358949 T1378 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 358950 T1378 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 358950 T1378 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 358950 T1378 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 358951 T1378 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 358951 T1378 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 358951 T1378 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 358952 T1378 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 358952 T1378 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 358953 T1378 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 358953 T1378 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 358954 T1378 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 358954 T1378 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 358955 T1378 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 358955 T1378 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 358955 T1378 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:37241/solr
   [junit4]   2> 358956 T1378 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 358956 T1378 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 358957 T1378 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 358960 T1449 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@32a9f9db name:ZooKeeperConnection Watcher:127.0.0.1:37241 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 358960 T1378 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 358961 T1378 oascc.SolrZkClient.createZ

[...truncated too long message...]

, but Solr cannot talk to ZK
   [junit4]   2> 1462700 T5785 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> 1462860 T5785 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:23622 23622
   [junit4]   2> 1463090 T5786 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:23622 23622
   [junit4]   2> 1463091 T5786 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		6	/solr/aliases.json
   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 		3	/solr/collections/c8n_1x3_commits/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent data watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		2	/solr/collections/c8n_1x3_commits/leader_elect/shard1/election/93534121648979980-core_node3-n_0000000000
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1463092 T5785 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:23702/
   [junit4]   2> 1463092 T5785 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:23638/
   [junit4]   2> 1463092 T5785 oasc.SocketProxy.close WARN Closing 11 connections to: http://127.0.0.1:23650/
   [junit4]   2> 1463092 T5785 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:23717/
   [junit4]   2> 1463093 T5785 oasc.SocketProxy.close WARN Closing 1 connections to: http://127.0.0.1:23684/
   [junit4]   2> 1463093 T5785 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:23702/
   [junit4]   2> 1463093 T5785 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:23638/
   [junit4]   2> 1463093 T5785 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:23650/
   [junit4]   2> 1463094 T5785 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:23717/
   [junit4]   2> 1463094 T5785 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:23684/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderInitiatedRecoveryOnCommitTest -Dtests.method=test -Dtests.seed=7C935111F5E670FD -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ar_IQ -Dtests.timezone=Africa/Gaborone -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   83.9s J3 | LeaderInitiatedRecoveryOnCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:23638/c8n_1x3_commits_shard1_replica2
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([7C935111F5E670FD:F4C76ECB5B1A1D05]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
   [junit4]    > 	... 48 more
   [junit4]   2> 1463110 T5785 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7C935111F5E670FD-001
   [junit4]   2> 83885 T5784 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=DefaultSimilarity, locale=ar_IQ, timezone=Africa/Gaborone
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=225058728,total=403177472
   [junit4]   2> NOTE: All tests run in this JVM: [TestClassNameShortening, SortByFunctionTest, TestReplicationHandler, CloudExitableDirectoryReaderTest, TestFaceting, TestPartialUpdateDeduplication, TestLRUCache, FastVectorHighlighterTest, TestCursorMarkWithoutUniqueKey, RequiredFieldsTest, DistributedFacetPivotSmallAdvancedTest, DefaultValueUpdateProcessorTest, TestStandardQParsers, TestPhraseSuggestions, MergeStrategyTest, RankQueryTest, TestDynamicFieldResource, WordBreakSolrSpellCheckerTest, AnalysisErrorHandlingTest, SpellCheckCollatorTest, TestSortingResponseWriter, RollingRestartTest, ShardRoutingCustomTest, SliceStateTest, PluginInfoTest, QueryEqualityTest, DistributedQueryElevationComponentTest, RequestHandlersTest, ReplicationFactorTest, BufferStoreTest, TestStressUserVersions, TestManagedResourceStorage, SoftAutoCommitTest, DocValuesMissingTest, FieldMutatingUpdateProcessorTest, TestOverriddenPrefixQueryForCustomFieldType, DeleteLastCustomShardedReplicaTest, BasicDistributedZk2Test, AssignTest, RecoveryAfterSoftCommitTest, PrimitiveFieldTypeTest, NumericFieldsTest, AnalyticsMergeStrategyTest, CustomCollectionTest, CoreMergeIndexesAdminHandlerTest, TestCoreContainer, ZkSolrClientTest, SOLR749Test, TestTrackingShardHandlerFactory, TestRawResponseWriter, TestIndexSearcher, SegmentsInfoRequestHandlerTest, AliasIntegrationTest, LeaderElectionTest, SchemaVersionSpecificBehaviorTest, JSONWriterTest, FacetPivotSmallTest, TestCloudManagedSchemaConcurrent, TestSolrConfigHandlerCloud, LeaderInitiatedRecoveryOnCommitTest]
   [junit4] Completed on J3 in 84.67s, 1 test, 1 error <<< FAILURES!

[...truncated 716 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 475 suites, 1894 tests, 3 errors, 62 ignored (22 assumptions)

Total time: 51 minutes 40 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 183892461 bytes
Compression is 0.0%
Took 34 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2826 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2826/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:28114/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28114/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([DD2F36EE1875D486:557B0934B689B97E]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:16496/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:16496/collection1
	at __randomizedtesting.SeedInfo.seed([DD2F36EE1875D486:557B0934B689B97E]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:16496/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:62691/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:62691/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([DD2F36EE1875D486:557B0934B689B97E]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:62691/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 53 more




Build Log:
[...truncated 9986 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/init-core-data-001
   [junit4]   2> 1388023 T4000 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 1388029 T4000 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1388030 T4001 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1388130 T4000 oasc.ZkTestServer.run start zk server on port:28079
   [junit4]   2> 1388131 T4000 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1388132 T4000 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1388135 T4008 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2df18288 name:ZooKeeperConnection Watcher:127.0.0.1:28079 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1388135 T4000 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1388136 T4000 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1388136 T4000 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1388139 T4000 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1388140 T4000 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1388141 T4011 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2b79aaea name:ZooKeeperConnection Watcher:127.0.0.1:28079/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1388141 T4000 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1388142 T4000 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1388142 T4000 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1388144 T4000 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1388146 T4000 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1388147 T4000 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1388149 T4000 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1388150 T4000 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1388152 T4000 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1388153 T4000 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1388255 T4000 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1388256 T4000 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1388258 T4000 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1388258 T4000 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1388260 T4000 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1388261 T4000 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1388263 T4000 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1388263 T4000 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1388265 T4000 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1388266 T4000 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1388268 T4000 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1388268 T4000 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1388270 T4000 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1388271 T4000 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1388273 T4000 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1388273 T4000 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1388275 T4000 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1388276 T4000 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1389921 T4000 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/control-001/cores/collection1
   [junit4]   2> 1389926 T4000 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1389931 T4000 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:?????
   [junit4]   2> 1389931 T4000 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=28084, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/control-001/cores, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/tempDir-001/control/data}
   [junit4]   2> 1389932 T4000 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 1389932 T4000 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/control-001/'
   [junit4]   2> 1389983 T4000 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/control-001/solr.xml
   [junit4]   2> 1390038 T4000 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/control-001/cores
   [junit4]   2> 1390039 T4000 oasc.CoreContainer.<init> New CoreContainer 789373592
   [junit4]   2> 1390039 T4000 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/control-001/]
   [junit4]   2> 1390039 T4000 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/control-001/lib
   [junit4]   2> 1390040 T4000 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/control-001/lib).
   [junit4]   2> 1390049 T4000 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1390049 T4000 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1390049 T4000 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1390050 T4000 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1390050 T4000 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1390050 T4000 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1390050 T4000 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1390050 T4000 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1390051 T4000 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1390051 T4000 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1390051 T4000 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1390052 T4000 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1390052 T4000 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1390052 T4000 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1390052 T4000 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1390052 T4000 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:28079/solr
   [junit4]   2> 1390053 T4000 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1390053 T4000 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1390054 T4000 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1390055 T4023 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4dd0b61d name:ZooKeeperConnection Watcher:127.0.0.1:28079 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1390056 T4000 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1390057 T4000 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1390059 T4000 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1390060 T4026 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6e3ecf61 name:ZooKeeperConnection Watcher:127.0.0.1:28079/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1390060 T4000 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1390062 T4000 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1390064 T4000 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1390066 T4000 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1390067 T4000 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1390069 T4000 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1390071 T4000 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1390073 T4000 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1390074 T4000 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1390075 T4000 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:28084_
   [junit4]   2> 1390076 T4000 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:28084_
   [junit4]   2> 1390078 T4000 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1390079 T4000 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1390080 T4000 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1390082 T4000 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:28084_
   [junit4]   2> 1390082 T4000 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1390084 T4000 oasc.Overseer.start Overseer (id=93533025439383555-127.0.0.1:28084_-n_0000000000) starting
   [junit4]   2> 1390085 T4000 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1390091 T4000 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 1390092 T4028 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1390092 T4000 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1390092 T4027 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1390096 T4000 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/control-001/cores
   [junit4]   2> 1390099 T4000 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=control_collection, shard=, transient=false, coreNodeName=, loadOnStartup=true, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/control-001/cores/collection1, schema=schema.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/control-001/cores/collection1/, config=solrconfig.xml, dataDir=data/, name=collection1}
   [junit4]   2> 1390100 T4000 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/control-001/cores/collection1/
   [junit4]   2> 1390100 T4000 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1390101 T4030 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1390101 T4030 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1390102 T4026 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1390102 T4030 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1390103 T4027 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28084",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:28084_",
   [junit4]   2> 	  "numShards":"1"} current state version: 0
   [junit4]   2> 1390103 T4027 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28084",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:28084_",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 1390103 T4027 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 1390104 T4027 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1390105 T4026 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1391102 T4030 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1391103 T4030 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1391103 T4030 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1391104 T4030 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1391104 T4030 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1391104 T4030 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/control-001/cores/collection1/'
   [junit4]   2> 1391118 T4030 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1391121 T4030 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1391131 T4030 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1391152 T4030 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 1391196 T4030 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1391297 T4030 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1391305 T4030 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1391657 T4030 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1391673 T4030 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1391676 T4030 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1391683 T4030 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1391687 T4030 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1391690 T4030 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1391691 T4030 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1391691 T4030 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1391691 T4030 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1391692 T4030 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1391692 T4030 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1391692 T4030 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1391692 T4030 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1391693 T4030 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/control-001/cores/collection1/data/
   [junit4]   2> 1391693 T4030 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@719a7b2c
   [junit4]   2> 1391693 T4030 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/control-001/cores/collection1/data
   [junit4]   2> 1391694 T4030 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/control-001/cores/collection1/data/index/
   [junit4]   2> 1391694 T4030 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1391694 T4030 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/control-001/cores/collection1/data/index
   [junit4]   2> 1391694 T4030 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=46, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5688173216882958]
   [junit4]   2> 1391695 T4030 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3380fb2e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@58eaa36c),segFN=segments_1,generation=1}
   [junit4]   2> 1391695 T4030 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1391699 T4030 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1391699 T4030 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1391699 T4030 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1391699 T4030 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1391699 T4030 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1391699 T4030 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1391700 T4030 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1391700 T4030 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1391700 T4030 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1391728 T4030 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1391730 T4030 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1391732 T4030 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1391734 T4030 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1391737 T4030 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /config,/update/json,standard,/update/json/docs,/get,/admin/mbeans,/admin/properties,/admin/segments,/update/csv,/replication,/admin/threads,/admin/ping,/admin/system,/update,/schema,/admin/logging,/admin/luke,/admin/plugins,/admin/file
   [junit4]   2> 1391739 T4030 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1391741 T4030 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1391741 T4030 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1391742 T4030 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1391743 T4030 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1391743 T4030 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=35, maxMergeAtOnceExplicit=31, maxMergedSegmentMB=71.0, floorSegmentMB=1.78125, forceMergeDeletesPctAllowed=3.7018674259254802, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1391743 T4030 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3380fb2e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@58eaa36c),segFN=segments_1,generation=1}
   [junit4]   2> 1391744 T4030 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1391744 T4030 oass.SolrIndexSearcher.<init> Opening Searcher@74e7d962[collection1] main
   [junit4]   2> 1391744 T4030 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1391745 T4030 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1391745 T4030 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1391745 T4030 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1391745 T4030 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1391746 T4030 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1391746 T4030 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1391746 T4030 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1391746 T4030 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1391746 T4030 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1391747 T4031 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@74e7d962[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1391747 T4030 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1391748 T4034 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:28084 collection:control_collection shard:shard1
   [junit4]   2> 1391748 T4000 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 1391748 T4000 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1391748 T4034 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1391751 T4000 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1391752 T4000 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1391752 T4034 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1391753 T4038 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@50212c05 name:ZooKeeperConnection Watcher:127.0.0.1:28079/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1391753 T4000 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1391754 T4000 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1391754 T4026 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1391754 T4034 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1391754 T4000 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1391754 T4034 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C1968 name=collection1 org.apache.solr.core.SolrCore@4f34d6cf url=http://127.0.0.1:28084/collection1 node=127.0.0.1:28084_ C1968_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:28084, core=collection1, state=down, node_name=127.0.0.1:28084_}
   [junit4]   2> 1391755 T4034 C1968 P28084 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:28084/collection1/
   [junit4]   2> 1391755 T4027 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 1391755 T4034 C1968 P28084 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1391756 T4034 C1968 P28084 oasc.SyncStrategy.syncToMe http://127.0.0.1:28084/collection1/ has no replicas
   [junit4]   2> 1391756 T4034 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:28084/collection1/ shard1
   [junit4]   2> 1391756 T4034 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1391756 T4026 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1391756 T4038 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1391757 T4000 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1391760 T4026 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1391761 T4027 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28084",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 1391864 T4026 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1391864 T4038 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1391913 T4034 oasc.ZkController.register We are http://127.0.0.1:28084/collection1/ and leader is http://127.0.0.1:28084/collection1/
   [junit4]   2> 1391914 T4034 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:28084
   [junit4]   2> 1391914 T4034 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1391914 T4034 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1391914 T4034 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1391915 T4026 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1391916 T4027 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28084",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:28084_",
   [junit4]   2> 	  "numShards":"1"} current state version: 3
   [junit4]   2> 1391917 T4027 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28084",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:28084_",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 1392019 T4026 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1392019 T4038 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1393373 T4000 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/shard-1-001/cores/collection1
   [junit4]   2> 1393377 T4000 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/shard-1-001
   [junit4]   2> 1393378 T4000 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1393381 T4000 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:?????
   [junit4]   2> 1393381 T4000 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solrconfig=solrconfig.xml, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/tempDir-001/jetty1, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/shard-1-001/cores, hostContext=/, hostPort=28092}
   [junit4]   2> 1393382 T4000 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 1393382 T4000 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/shard-1-001/'
   [junit4]   2> 1393435 T4000 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/shard-1-001/solr.xml
   [junit4]   2> 1393499 T4000 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/shard-1-001/cores
   [junit4]   2> 1393499 T4000 oasc.CoreContainer.<init> New CoreContainer 1303511738
   [junit4]   2> 1393500 T4000 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/shard-1-001/]
   [junit4]   2> 1393500 T4000 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/shard-1-001/lib
   [junit4]   2> 1393501 T4000 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/shard-1-001/lib).
   [junit4]   2> 1393515 T4000 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1393516 T4000 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1393516 T4000 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1393517 T4000 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1393517 T4000 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1393517 T4000 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1393517 T4000 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1393518 T4000 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1393518 T4000 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1393518 T4000 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1393519 T4000 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1393519 T4000 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1393520 T4000 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1393520 T4000 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1393520 T4000 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1393521 T4000 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:28079/solr
   [junit4]   2> 1393521 T4000 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1393521 T4000 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1393522 T4000 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1393524 T4050 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4e23e38f name:ZooKeeperConnection Watcher:127.0.0.1:28079 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1393525 T4000 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1393526 T4000 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1393528 T4000 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1393529 T4053 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@31bee5ff name:ZooKeeperConnection Watcher:127.0.0.1:28079/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1393529 T4000 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1393533 T4000 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1394537 T4000 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:28092_
   [junit4]   2> 1394538 T4000 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:28092_
   [junit4]   2> 1394541 T4000 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1394543 T4000 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/shard-1-001/cores
   [junit4]   2> 1394546 T4000 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {coreNodeName=, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/shard-1-001/cores/collection1, loadOnStartup=true, schema=schema.xml, dataDir=data/, collection=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/shard-1-001/cores/collection1/, shard=, transient=false, name=collection1}
   [junit4]   2> 1394546 T4000 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/shard-1-001/cores/collection1/
   [junit4]   2> 1394547 T4000 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1394548 T4054 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1394548 T4054 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1394548 T4054 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1394548 T4026 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1394549 T4027 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28092",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:28092_",
   [junit4]   2> 	  "numShards":"1"} current state version: 4
   [junit4]   2> 1394550 T4027 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28092",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:28092_",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 1394550 T4027 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 1394550 T4027 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1394551 T4053 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1394552 T4038 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1394551 T4026 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1395549 T4054 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1395550 T4054 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1395550 T4054 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1395551 T4054 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1395551 T4054 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1395551 T4054 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/shard-1-001/cores/collection1/'
   [junit4]   2> 1395570 T4054 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1395572 T4054 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1395585 T4054 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1395612 T4054 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 1395681 T4054 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1395782 T4054 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1395790 T4054 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1396114 T4054 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1396126 T4054 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1396128 T4054 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1396137 T4054 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1396141 T4054 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1396144 T4054 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1396145 T4054 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1396145 T4054 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1396145 T4054 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1396146 T4054 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1396146 T4054 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1396146 T4054 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1396146 T4054 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1396146 T4054 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 1396147 T4054 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@719a7b2c
   [junit4]   2> 1396147 T4054 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/shard-1-001/cores/collection1/data
   [junit4]   2> 1396147 T4054 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 1396148 T4054 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1396148 T4054 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 1396148 T4054 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=46, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5688173216882958]
   [junit4]   2> 1396149 T4054 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6f159815 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@72b3c3ac),segFN=segments_1,generation=1}
   [junit4]   2> 1396149 T4054 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1396153 T4054 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1396154 T4054 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1396154 T4054 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1396154 T4054 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1396154 T4054 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1396154 T4054 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1396154 T4054 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1396155 T4054 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1396155 T4054 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1396170 T4054 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1396172 T4054 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1396173 T4054 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1396175 T4054 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1396180 T4054 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /config,/update/json,standard,/update/json/docs,/get,/admin/mbeans,/admin/properties,/admin/segments,/update/csv,/replication,/admin/threads,/admin/ping,/admin/system,/update,/schema,/admin/logging,/admin/luke,/admin/plugins,/admin/file
   [junit4]   2> 1396181 T4054 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1396183 T4054 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1396183 T4054 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1396184 T4054 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1396184 T4054 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1396185 T4054 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=35, maxMergeAtOnceExplicit=31, maxMergedSegmentMB=71.0, floorSegmentMB=1.78125, forceMergeDeletesPctAllowed=3.7018674259254802, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1396186 T4054 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6f159815 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@72b3c3ac),segFN=segments_1,generation=1}
   [junit4]   2> 1396186 T4054 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1396186 T4054 oass.SolrIndexSearcher.<init> Opening Searcher@56dafdcd[collection1] main
   [junit4]   2> 1396186 T4054 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1396187 T4054 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1396187 T4054 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1396188 T4054 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1396188 T4054 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1396188 T4054 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1396188 T4054 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1396189 T4054 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1396189 T4054 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1396189 T4054 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1396190 T4055 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@56dafdcd[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1396190 T4054 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1396191 T4058 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:28092 collection:collection1 shard:shard1
   [junit4]   2> 1396191 T4000 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3
   [junit4]   2> 1396192 T4000 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1396192 T4058 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1396197 T4058 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1396198 T4026 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1396198 T4058 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1396199 T4058 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C1969 name=collection1 org.apache.solr.core.SolrCore@5bc4ab2d url=http://127.0.0.1:28092/collection1 node=127.0.0.1:28092_ C1969_STATE=coll:collection1 core:collection1 props:{base_url=http://127.0.0.1:28092, core=collection1, state=down, node_name=127.0.0.1:28092_}
   [junit4]   2> 1396199 T4058 C1969 P28092 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:28092/collection1/
   [junit4]   2> 1396199 T4027 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 1396199 T4058 C1969 P28092 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1396200 T4058 C1969 P28092 oasc.SyncStrategy.syncToMe http://127.0.0.1:28092/collection1/ has no replicas
   [junit4]   2> 1396200 T4058 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:28092/collection1/ shard1
   [junit4]   2> 1396200 T4058 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1396200 T4026 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1396201 T4053 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1396200 T4038 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1396204 T4026 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1396205 T4027 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28092",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 1396308 T4026 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1396308 T4038 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1396308 T4053 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1396357 T4058 oasc.ZkController.register We are http://127.0.0.1:28092/collection1/ and leader is http://127.0.0.1:28092/collection1/
   [junit4]   2> 1396358 T4058 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:28092
   [junit4]   2> 1396358 T4058 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1396358 T4058 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 1396358 T4058 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1396360 T4026 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1396360 T4027 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28092",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:28092_",
   [junit4]   2> 	  "numShards":"1"} current state version: 7
   [junit4]   2> 1396361 T4027 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28092",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:28092_",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 1396464 T4026 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1396464 T4038 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1396464 T4053 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1397639 T4000 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/shard-2-001/cores/collection1
   [junit4]   2> 1397641 T4000 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/shard-2-001
   [junit4]   2> 1397642 T4000 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1397644 T4000 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:?????
   [junit4]   2> 1397644 T4000 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/tempDir-001/jetty2, hostPort=28101, hostContext=/, solrconfig=solrconfig.xml, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/shard-2-001/cores}
   [junit4]   2> 1397644 T4000 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 1397645 T4000 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/shard-2-001/'
   [junit4]   2> 1397682 T4000 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/shard-2-001/solr.xml
   [junit4]   2> 1397729 T4000 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/shard-2-001/cores
   [junit4]   2> 1397729 T4000 oasc.CoreContainer.<init> New CoreContainer 239336256
   [junit4]   2> 1397730 T4000 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/shard-2-001/]
   [junit4]   2> 1397730 T4000 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/shard-2-001/lib
   [junit4]   2> 1397731 T4000 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DD2F36EE1875D486-001/shard-2-001/lib).
   [junit4]   2> 1397740 T4000 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1397741 T4000 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1397741 T4000 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1397741 T4000 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1397741 T4000 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1397741 T4000 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1397742 T4000 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1397742 T4000 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1397742 T4000 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1397742 T4000 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1397743 T4000 oashc.HttpShardHandlerFactory.getParameter Setting useRe

[...truncated too long message...]

Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1555483 T4173 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16487/
   [junit4]   2> 1555483 T4173 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:16503/
   [junit4]   2> 1555483 T4173 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:16496/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=DD2F36EE1875D486 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=bg -Dtests.timezone=Africa/Bujumbura -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   62.7s J2 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:16496/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([DD2F36EE1875D486:557B0934B689B97E]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:16496/collection1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 49 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
   [junit4]    > 	... 54 more
   [junit4]   2> 1555497 T4173 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest DD2F36EE1875D486-001
   [junit4]   2> 62746 T4172 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {timestamp=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), a_t=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), intDefault=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), multiDefault=PostingsFormat(name=LuceneVarGapFixedInterval), range_facet_l=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), id=FST50, _version_=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), text=PostingsFormat(name=LuceneFixedGap)}, docValues:{timestamp=DocValuesFormat(name=Asserting)}, sim=DefaultSimilarity, locale=bg, timezone=Africa/Bujumbura
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=182268024,total=346554368
   [junit4]   2> NOTE: All tests run in this JVM: [ClusterStateUpdateTest, TestReplicaProperties, TestCloudInspectUtil, FileBasedSpellCheckerTest, UpdateParamsTest, DistanceFunctionTest, TestPostingsSolrHighlighter, RegexBoostProcessorTest, TestPivotHelperCode, TestExactStatsCache, HdfsChaosMonkeySafeLeaderTest, TestSurroundQueryParser, AtomicUpdatesTest, ShardRoutingCustomTest, TestIBSimilarityFactory, IndexSchemaTest, ClusterStateTest, SolrCoreTest, SpatialRPTFieldTypeTest, DirectUpdateHandlerOptimizeTest, TestRecoveryHdfs, SuggesterTSTTest, AddSchemaFieldsUpdateProcessorFactoryTest, OpenExchangeRatesOrgProviderTest, DistribCursorPagingTest, TestRecovery, TestReloadAndDeleteDocs, DistributedDebugComponentTest, TestStressLucene, SolrPluginUtilsTest, TestShortCircuitedRequests, TestAtomicUpdateErrorCases, ChaosMonkeySafeLeaderTest, TestDistributedSearch, TestCloudPivotFacet, TestQuerySenderListener, DocValuesMissingTest, TestNamedUpdateProcessors, SuggestComponentTest, SolrCoreCheckLockOnStartupTest, CustomCollectionTest, DistributedQueueTest, CacheHeaderTest, TestExactSharedStatsCache, TestPseudoReturnFields, RequestLoggingTest, TestSchemaVersionResource, TestDFRSimilarityFactory, DeleteReplicaTest, CoreAdminCreateDiscoverTest, TestBulkSchemaConcurrent, HdfsWriteToMultipleCollectionsTest, ZkStateWriterTest, ConnectionManagerTest, TestImplicitCoreProperties, TestRandomFaceting, TermsComponentTest, LeaderFailoverAfterPartitionTest, TestSolrConfigHandlerCloud, ReplicationFactorTest, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J2 in 63.54s, 1 test, 1 error <<< FAILURES!

[...truncated 691 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 475 suites, 1892 tests, 3 errors, 62 ignored (22 assumptions)

Total time: 52 minutes 19 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 188707626 bytes
Compression is 0.0%
Took 36 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2825 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2825/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:10419/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:10419/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([D6FD21FC23D337CE:5EA91E268D2F5A36]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:10686/x/k/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:10686/x/k/collection1
	at __randomizedtesting.SeedInfo.seed([D6FD21FC23D337CE:5EA91E268D2F5A36]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:10686/x/k/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28619/nblos/h/repfacttest_c8n_1x3_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28619/nblos/h/repfacttest_c8n_1x3_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([D6FD21FC23D337CE:5EA91E268D2F5A36]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28619/nblos/h/repfacttest_c8n_1x3_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 53 more




Build Log:
[...truncated 9893 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/init-core-data-001
   [junit4]   2> 962190 T2056 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /nblos/h
   [junit4]   2> 962196 T2056 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 962197 T2057 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 962297 T2056 oasc.ZkTestServer.run start zk server on port:28598
   [junit4]   2> 962298 T2056 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 962299 T2056 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 962303 T2064 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3ef860b6 name:ZooKeeperConnection Watcher:127.0.0.1:28598 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 962303 T2056 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 962304 T2056 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 962304 T2056 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 962306 T2056 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 962307 T2056 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 962308 T2067 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@8c2e416 name:ZooKeeperConnection Watcher:127.0.0.1:28598/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 962309 T2056 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 962309 T2056 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 962309 T2056 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 962311 T2056 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 962313 T2056 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 962314 T2056 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 962316 T2056 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 962317 T2056 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 962320 T2056 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 962321 T2056 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 962323 T2056 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 962324 T2056 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 962326 T2056 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 962327 T2056 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 962329 T2056 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 962329 T2056 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 962331 T2056 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 962332 T2056 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 962334 T2056 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 962335 T2056 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 962337 T2056 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 962338 T2056 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 962340 T2056 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 962341 T2056 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 962343 T2056 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 962343 T2056 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 962345 T2056 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 962346 T2056 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 964025 T2056 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/control-001/cores/collection1
   [junit4]   2> 964028 T2056 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 964030 T2056 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:28603
   [junit4]   2> 964030 T2056 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/tempDir-001/control/data, hostPort=28602, hostContext=/nblos/h, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/control-001/cores}
   [junit4]   2> 964031 T2056 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 964031 T2056 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/control-001/'
   [junit4]   2> 964062 T2056 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/control-001/solr.xml
   [junit4]   2> 964100 T2056 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/control-001/cores
   [junit4]   2> 964100 T2056 oasc.CoreContainer.<init> New CoreContainer 1631586395
   [junit4]   2> 964100 T2056 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/control-001/]
   [junit4]   2> 964101 T2056 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/control-001/lib
   [junit4]   2> 964101 T2056 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/control-001/lib).
   [junit4]   2> 964110 T2056 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 964111 T2056 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 964111 T2056 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 964111 T2056 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 964111 T2056 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 964112 T2056 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 964112 T2056 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 964112 T2056 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 964112 T2056 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 964112 T2056 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 964113 T2056 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 964113 T2056 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 964113 T2056 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 964114 T2056 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 964114 T2056 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 964114 T2056 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:28598/solr
   [junit4]   2> 964114 T2056 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 964114 T2056 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 964115 T2056 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 964117 T2079 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5d21c88c name:ZooKeeperConnection Watcher:127.0.0.1:28598 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 964117 T2056 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 964118 T2056 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 964134 T2056 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 964135 T2082 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2b5dc903 name:ZooKeeperConnection Watcher:127.0.0.1:28598/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 964136 T2056 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 964137 T2056 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 964139 T2056 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 964140 T2056 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 964142 T2056 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 964144 T2056 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 964145 T2056 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 964147 T2056 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 964148 T2056 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 964149 T2056 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:28602_nblos%2Fh
   [junit4]   2> 964149 T2056 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:28602_nblos%2Fh
   [junit4]   2> 964151 T2056 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 964152 T2056 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 964153 T2056 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 964154 T2056 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:28602_nblos%2Fh
   [junit4]   2> 964154 T2056 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 964156 T2056 oasc.Overseer.start Overseer (id=93532051036438531-127.0.0.1:28602_nblos%2Fh-n_0000000000) starting
   [junit4]   2> 964157 T2056 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 964162 T2056 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 964163 T2084 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 964163 T2056 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 964163 T2083 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 964167 T2056 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/control-001/cores
   [junit4]   2> 964169 T2056 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {coreNodeName=, config=solrconfig.xml, name=collection1, transient=false, shard=, collection=control_collection, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/control-001/cores/collection1/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/control-001/cores/collection1, loadOnStartup=true, dataDir=data/, schema=schema.xml}
   [junit4]   2> 964169 T2056 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/control-001/cores/collection1/
   [junit4]   2> 964169 T2056 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 964170 T2086 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 964170 T2086 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 964171 T2086 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 964171 T2082 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 964172 T2083 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:28602_nblos%2Fh",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28602/nblos/h",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1"} current state version: 0
   [junit4]   2> 964172 T2083 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:28602_nblos%2Fh",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28602/nblos/h",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 964172 T2083 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 964172 T2083 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 964173 T2082 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 965171 T2086 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 965172 T2086 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 965172 T2086 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 965172 T2086 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 965173 T2086 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 965173 T2086 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/control-001/cores/collection1/'
   [junit4]   2> 965186 T2086 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 965189 T2086 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 965198 T2086 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 965217 T2086 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 965258 T2086 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 965259 T2086 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 965267 T2086 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 965585 T2086 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 965598 T2086 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 965600 T2086 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 965610 T2086 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 965614 T2086 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 965617 T2086 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 965618 T2086 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 965618 T2086 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 965618 T2086 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 965619 T2086 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 965619 T2086 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 965619 T2086 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 965619 T2086 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 965620 T2086 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/control-001/cores/collection1/data/
   [junit4]   2> 965620 T2086 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b292965
   [junit4]   2> 965620 T2086 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/control-001/cores/collection1/data
   [junit4]   2> 965621 T2086 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/control-001/cores/collection1/data/index/
   [junit4]   2> 965621 T2086 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 965621 T2086 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/control-001/cores/collection1/data/index
   [junit4]   2> 965621 T2086 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=31, maxMergeAtOnceExplicit=22, maxMergedSegmentMB=31.0712890625, floorSegmentMB=1.02734375, forceMergeDeletesPctAllowed=27.304524387613643, segmentsPerTier=42.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 965622 T2086 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@641979d5 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@448dc24a),segFN=segments_1,generation=1}
   [junit4]   2> 965622 T2086 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 965626 T2086 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 965626 T2086 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 965627 T2086 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 965627 T2086 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 965627 T2086 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 965627 T2086 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 965627 T2086 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 965628 T2086 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 965628 T2086 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 965642 T2086 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 965643 T2086 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 965645 T2086 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 965646 T2086 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 965650 T2086 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/segments,/admin/mbeans,/admin/system,/replication,/admin/logging,/update/json/docs,/schema,/config,/update/json,/admin/luke,/admin/ping,/admin/properties,/update,/admin/plugins,/update/csv,/admin/threads,/admin/file,/get,standard
   [junit4]   2> 965651 T2086 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 965652 T2086 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 965653 T2086 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 965654 T2086 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 965654 T2086 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 965654 T2086 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=29, maxMergeAtOnceExplicit=39, maxMergedSegmentMB=77.9013671875, floorSegmentMB=1.6181640625, forceMergeDeletesPctAllowed=16.45488476462364, segmentsPerTier=14.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6937156718775733
   [junit4]   2> 965655 T2086 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@641979d5 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@448dc24a),segFN=segments_1,generation=1}
   [junit4]   2> 965655 T2086 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 965655 T2086 oass.SolrIndexSearcher.<init> Opening Searcher@5ef0941c[collection1] main
   [junit4]   2> 965655 T2086 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 965656 T2086 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 965656 T2086 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 965657 T2086 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 965657 T2086 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 965657 T2086 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 965658 T2086 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 965658 T2086 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 965658 T2086 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 965658 T2086 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 965659 T2086 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 965659 T2087 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5ef0941c[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 965659 T2090 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:28602/nblos/h collection:control_collection shard:shard1
   [junit4]   2> 965660 T2056 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 965660 T2056 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 965660 T2090 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 965661 T2056 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 965661 T2056 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 965663 T2094 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@749a1c6a name:ZooKeeperConnection Watcher:127.0.0.1:28598/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 965663 T2056 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 965663 T2056 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 965663 T2056 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 965665 T2090 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 965665 T2056 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 965666 T2082 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 965666 T2090 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 965667 T2090 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C343 name=collection1 org.apache.solr.core.SolrCore@50e92c91 url=http://127.0.0.1:28602/nblos/h/collection1 node=127.0.0.1:28602_nblos%2Fh C343_STATE=coll:control_collection core:collection1 props:{core=collection1, node_name=127.0.0.1:28602_nblos%2Fh, base_url=http://127.0.0.1:28602/nblos/h, state=down}
   [junit4]   2> 965667 T2090 C343 P28602 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:28602/nblos/h/collection1/
   [junit4]   2> 965667 T2090 C343 P28602 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 965667 T2083 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 965667 T2090 C343 P28602 oasc.SyncStrategy.syncToMe http://127.0.0.1:28602/nblos/h/collection1/ has no replicas
   [junit4]   2> 965667 T2090 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:28602/nblos/h/collection1/ shard1
   [junit4]   2> 965668 T2090 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 965671 T2082 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 965672 T2083 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28602/nblos/h",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 1
   [junit4]   2> 965775 T2094 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 965775 T2082 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 965824 T2090 oasc.ZkController.register We are http://127.0.0.1:28602/nblos/h/collection1/ and leader is http://127.0.0.1:28602/nblos/h/collection1/
   [junit4]   2> 965825 T2090 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:28602/nblos/h
   [junit4]   2> 965825 T2090 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 965825 T2090 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 965825 T2090 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 965827 T2082 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 965827 T2083 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:28602_nblos%2Fh",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28602/nblos/h",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"3"} current state version: 2
   [junit4]   2> 965828 T2083 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:28602_nblos%2Fh",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28602/nblos/h",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"3"}
   [junit4]   2> 965931 T2082 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 965931 T2094 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 966959 T2056 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/shard-1-001/cores/collection1
   [junit4]   2> 966961 T2056 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/shard-1-001
   [junit4]   2> 966962 T2056 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 966964 T2056 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:28609
   [junit4]   2> 966965 T2056 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solrconfig=solrconfig.xml, hostContext=/nblos/h, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/tempDir-001/jetty1, hostPort=28608, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/shard-1-001/cores}
   [junit4]   2> 966965 T2056 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 966965 T2056 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/shard-1-001/'
   [junit4]   2> 966998 T2056 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/shard-1-001/solr.xml
   [junit4]   2> 967037 T2056 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/shard-1-001/cores
   [junit4]   2> 967037 T2056 oasc.CoreContainer.<init> New CoreContainer 1420067397
   [junit4]   2> 967038 T2056 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/shard-1-001/]
   [junit4]   2> 967038 T2056 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/shard-1-001/lib
   [junit4]   2> 967038 T2056 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/shard-1-001/lib).
   [junit4]   2> 967047 T2056 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 967047 T2056 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 967048 T2056 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 967048 T2056 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 967048 T2056 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 967048 T2056 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 967048 T2056 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 967049 T2056 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 967049 T2056 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 967049 T2056 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 967049 T2056 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 967050 T2056 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 967050 T2056 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 967050 T2056 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 967050 T2056 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 967051 T2056 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:28598/solr
   [junit4]   2> 967051 T2056 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 967051 T2056 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 967052 T2056 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 967053 T2106 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4c7c73c9 name:ZooKeeperConnection Watcher:127.0.0.1:28598 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 967054 T2056 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 967054 T2056 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 967056 T2056 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 967057 T2109 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7c5c1a75 name:ZooKeeperConnection Watcher:127.0.0.1:28598/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 967057 T2056 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 967060 T2056 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 968064 T2056 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:28608_nblos%2Fh
   [junit4]   2> 968065 T2056 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:28608_nblos%2Fh
   [junit4]   2> 968068 T2056 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 968070 T2056 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/shard-1-001/cores
   [junit4]   2> 968071 T2056 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {loadOnStartup=true, collection=collection1, dataDir=data/, name=collection1, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/shard-1-001/cores/collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/shard-1-001/cores/collection1/, config=solrconfig.xml, shard=, transient=false, coreNodeName=}
   [junit4]   2> 968072 T2056 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/shard-1-001/cores/collection1/
   [junit4]   2> 968072 T2056 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 968073 T2110 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 968073 T2110 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 968074 T2110 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 968074 T2082 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 968075 T2083 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:28608_nblos%2Fh",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28608/nblos/h",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"3"} current state version: 3
   [junit4]   2> 968075 T2083 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:28608_nblos%2Fh",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28608/nblos/h",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"3"}
   [junit4]   2> 968075 T2083 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 968075 T2083 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard3
   [junit4]   2> 968077 T2082 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 968077 T2094 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 968077 T2109 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 969074 T2110 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 969075 T2110 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 969075 T2110 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 969076 T2110 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 969076 T2110 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 969077 T2110 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/shard-1-001/cores/collection1/'
   [junit4]   2> 969095 T2110 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 969097 T2110 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 969110 T2110 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 969135 T2110 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 969191 T2110 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 969193 T2110 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 969204 T2110 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 969523 T2110 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 969535 T2110 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 969537 T2110 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 969546 T2110 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 969549 T2110 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 969552 T2110 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 969553 T2110 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 969553 T2110 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 969553 T2110 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 969554 T2110 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 969554 T2110 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 969554 T2110 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 969554 T2110 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 969554 T2110 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 969555 T2110 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b292965
   [junit4]   2> 969555 T2110 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/shard-1-001/cores/collection1/data
   [junit4]   2> 969556 T2110 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 969556 T2110 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 969556 T2110 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 969556 T2110 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=31, maxMergeAtOnceExplicit=22, maxMergedSegmentMB=31.0712890625, floorSegmentMB=1.02734375, forceMergeDeletesPctAllowed=27.304524387613643, segmentsPerTier=42.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 969557 T2110 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@57531931 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@109b0cc1),segFN=segments_1,generation=1}
   [junit4]   2> 969557 T2110 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 969562 T2110 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 969562 T2110 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 969562 T2110 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 969562 T2110 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 969563 T2110 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 969563 T2110 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 969563 T2110 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 969563 T2110 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 969563 T2110 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 969577 T2110 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 969579 T2110 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 969581 T2110 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 969582 T2110 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 969588 T2110 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/segments,/admin/mbeans,/admin/system,/replication,/admin/logging,/update/json/docs,/schema,/config,/update/json,/admin/luke,/admin/ping,/admin/properties,/update,/admin/plugins,/update/csv,/admin/threads,/admin/file,/get,standard
   [junit4]   2> 969590 T2110 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 969591 T2110 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 969591 T2110 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 969593 T2110 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 969593 T2110 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 969594 T2110 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=29, maxMergeAtOnceExplicit=39, maxMergedSegmentMB=77.9013671875, floorSegmentMB=1.6181640625, forceMergeDeletesPctAllowed=16.45488476462364, segmentsPerTier=14.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6937156718775733
   [junit4]   2> 969594 T2110 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@57531931 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@109b0cc1),segFN=segments_1,generation=1}
   [junit4]   2> 969595 T2110 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 969595 T2110 oass.SolrIndexSearcher.<init> Opening Searcher@7920c65f[collection1] main
   [junit4]   2> 969595 T2110 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 969596 T2110 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 969596 T2110 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 969596 T2110 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 969597 T2110 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 969597 T2110 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 969597 T2110 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 969597 T2110 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 969598 T2110 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 969598 T2110 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 969599 T2111 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7920c65f[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 969599 T2110 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 969599 T2114 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:28608/nblos/h collection:collection1 shard:shard3
   [junit4]   2> 969600 T2056 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 969600 T2056 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 969600 T2114 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard3/election
   [junit4]   2> 969604 T2114 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard3
   [junit4]   2> 969606 T2082 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 969606 T2114 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 969606 T2114 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C344 name=collection1 org.apache.solr.core.SolrCore@4c81b2a4 url=http://127.0.0.1:28608/nblos/h/collection1 node=127.0.0.1:28608_nblos%2Fh C344_STATE=coll:collection1 core:collection1 props:{core=collection1, node_name=127.0.0.1:28608_nblos%2Fh, base_url=http://127.0.0.1:28608/nblos/h, state=down}
   [junit4]   2> 969606 T2114 C344 P28608 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:28608/nblos/h/collection1/
   [junit4]   2> 969606 T2114 C344 P28608 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 969607 T2083 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1"} current state version: 4
   [junit4]   2> 969607 T2114 C344 P28608 oasc.SyncStrategy.syncToMe http://127.0.0.1:28608/nblos/h/collection1/ has no replicas
   [junit4]   2> 969607 T2114 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:28608/nblos/h/collection1/ shard3
   [junit4]   2> 969607 T2114 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard3
   [junit4]   2> 969608 T2082 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 969609 T2094 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 969608 T2109 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 969612 T2083 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28608/nblos/h",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 969614 T2082 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 969615 T2109 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 969615 T2094 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 969615 T2082 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 969662 T2114 oasc.ZkController.register We are http://127.0.0.1:28608/nblos/h/collection1/ and leader is http://127.0.0.1:28608/nblos/h/collection1/
   [junit4]   2> 969663 T2114 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:28608/nblos/h
   [junit4]   2> 969663 T2114 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 969663 T2114 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 969663 T2114 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 969664 T2082 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 969666 T2083 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "node_name":"127.0.0.1:28608_nblos%2Fh",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28608/nblos/h",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"3"} current state version: 6
   [junit4]   2> 969666 T2083 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "node_name":"127.0.0.1:28608_nblos%2Fh",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28608/nblos/h",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"3"}
   [junit4]   2> 969770 T2082 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 969770 T2109 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 969770 T2094 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 970670 T2056 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/shard-2-001/cores/collection1
   [junit4]   2> 970672 T2056 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/shard-2-001
   [junit4]   2> 970673 T2056 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 970676 T2056 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:28620
   [junit4]   2> 970676 T2056 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=28619, hostContext=/nblos/h, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/shard-2-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/tempDir-001/jetty2, solrconfig=solrconfig.xml}
   [junit4]   2> 970676 T2056 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 970677 T2056 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/shard-2-001/'
   [junit4]   2> 970709 T2056 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/shard-2-001/solr.xml
   [junit4]   2> 970747 T2056 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/shard-2-001/cores
   [junit4]   2> 970748 T2056 oasc.CoreContainer.<init> New CoreContainer 501968293
   [junit4]   2> 970748 T2056 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/shard-2-001/]
   [junit4]   2> 970748 T2056 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/shard-2-001/lib
   [junit4]   2> 970749 T2056 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest D6FD21FC23D337CE-001/shard-2-001/lib).
   [junit4]   2> 970758 T2056 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 970759 T2056 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 970759 T2056 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 970759 T2056 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 970759 T2056 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 970760 T2056 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 970760 T2056 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 970760 T2056 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 970760 T2056 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 970760 T2056 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 970761 T2056 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 970761 T2056 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 970761 T2056 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 970762 T2056 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 970762 T2056 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 970762 T2056 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:28598/solr
   [junit4]   2> 970762 T2056 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 970763 T2056 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 970763 T2056 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 970765 T2127 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4b623ffd name:ZooKeeperConnection Watcher:127.0.0.1

[...truncated too long message...]

tServer.send4LetterWord connecting to 127.0.0.1:10670 10670
   [junit4]   2> 1517500 T3416 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:10670 10670
   [junit4]   2> 1517501 T3416 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1517501 T3415 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:10674/x/k
   [junit4]   2> 1517502 T3415 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:10686/x/k
   [junit4]   2> 1517502 T3415 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:10711/x/k
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=D6FD21FC23D337CE -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=nl_NL -Dtests.timezone=US/Samoa -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   62.0s J1 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:10686/x/k/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([D6FD21FC23D337CE:5EA91E268D2F5A36]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:10686/x/k/collection1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 49 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
   [junit4]    > 	... 54 more
   [junit4]   2> 1517510 T3415 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.RecoveryAfterSoftCommitTest D6FD21FC23D337CE-001
   [junit4]   2> 61980 T3414 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {id=PostingsFormat(name=LuceneVarGapFixedInterval), intDefault=PostingsFormat(name=Memory doPackFST= true), a_t=PostingsFormat(name=Direct), text=PostingsFormat(name=LuceneFixedGap), timestamp=PostingsFormat(name=Direct), _version_=PostingsFormat(name=Memory doPackFST= true), range_facet_l=PostingsFormat(name=Direct), multiDefault=PostingsFormat(name=Direct)}, docValues:{timestamp=DocValuesFormat(name=Lucene50)}, sim=DefaultSimilarity, locale=nl_NL, timezone=US/Samoa
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=106958896,total=313524224
   [junit4]   2> NOTE: All tests run in this JVM: [SolrTestCaseJ4Test, TestOrdValues, DocValuesMultiTest, TestRequestStatusCollectionAPI, SimplePostToolTest, TestCharFilters, CachingDirectoryFactoryTest, TestAnalyzedSuggestions, QueryElevationComponentTest, TestRemoteStreaming, TestSortingResponseWriter, TestReload, TestRealTimeGet, TestSuggestSpellingConverter, DebugComponentTest, CoreAdminRequestStatusTest, HighlighterConfigTest, TestIBSimilarityFactory, PolyFieldTest, TestCSVLoader, TestSimpleQParserPlugin, HdfsBasicDistributedZk2Test, TestUniqueKeyFieldResource, TestRandomMergePolicy, TestManagedSchema, LeaderElectionTest, DistributedQueryElevationComponentTest, TestDocumentBuilder, PingRequestHandlerTest, DeleteLastCustomShardedReplicaTest, TestSchemaManager, ShardSplitTest, TestFoldingMultitermQuery, PathHierarchyTokenizerFactoryTest, DefaultValueUpdateProcessorTest, TestRangeQuery, DirectUpdateHandlerTest, TestFuzzyAnalyzedSuggestions, BasicZkTest, LeaderFailoverAfterPartitionTest, IndexSchemaTest, LoggingHandlerTest, TestJmxIntegration, TestWriterPerf, TestSchemaResource, DocumentAnalysisRequestHandlerTest, ConvertedLegacyTest, MultiTermTest, DistributedQueryComponentOptimizationTest, DistributedMLTComponentTest, ReplicationFactorTest, DocValuesTest, DirectSolrConnectionTest, TestExactStatsCache, SortByFunctionTest, TestDistribDocBasedVersion, TestCloudSchemaless, ChaosMonkeyNothingIsSafeTest, SpatialHeatmapFacetsTest, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J1 in 62.51s, 1 test, 1 error <<< FAILURES!

[...truncated 696 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 475 suites, 1892 tests, 3 errors, 62 ignored (22 assumptions)

Total time: 51 minutes 35 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 198964011 bytes
Compression is 0.0%
Took 40 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2824 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2824/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:17611/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:17611/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([AE25518C57560B2C:26716E56F9AA66D4]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:14578/nltls/lk/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:14578/nltls/lk/collection1
	at __randomizedtesting.SeedInfo.seed([AE25518C57560B2C:26716E56F9AA66D4]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:14578/nltls/lk/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:41050/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:41050/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([AE25518C57560B2C:26716E56F9AA66D4]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:41050/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 53 more




Build Log:
[...truncated 9422 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/init-core-data-001
   [junit4]   2> 213003 T516 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 213010 T516 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 213011 T517 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 213111 T516 oasc.ZkTestServer.run start zk server on port:13401
   [junit4]   2> 213112 T516 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 213113 T516 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 213117 T524 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@30ec75ee name:ZooKeeperConnection Watcher:127.0.0.1:13401 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 213118 T516 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 213118 T516 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 213119 T516 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 213122 T516 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 213123 T516 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 213124 T527 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@710d3073 name:ZooKeeperConnection Watcher:127.0.0.1:13401/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 213125 T516 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 213125 T516 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 213126 T516 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 213128 T516 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 213130 T516 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 213131 T516 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 213133 T516 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 213134 T516 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 213137 T516 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 213138 T516 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 213140 T516 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 213140 T516 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 213142 T516 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 213143 T516 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 213145 T516 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 213145 T516 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 213150 T516 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 213151 T516 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 213153 T516 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 213153 T516 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 213155 T516 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 213156 T516 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 213163 T516 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 213163 T516 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 213166 T516 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 213166 T516 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 213168 T516 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 213169 T516 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 215648 T516 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/control-001/cores/collection1
   [junit4]   2> 215653 T516 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 215658 T516 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:57844
   [junit4]   2> 215659 T516 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/control-001/cores, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/tempDir-001/control/data, hostPort=54988}
   [junit4]   2> 215660 T516 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 215660 T516 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/control-001/'
   [junit4]   2> 215715 T516 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/control-001/solr.xml
   [junit4]   2> 215797 T516 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/control-001/cores
   [junit4]   2> 215798 T516 oasc.CoreContainer.<init> New CoreContainer 832564927
   [junit4]   2> 215798 T516 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/control-001/]
   [junit4]   2> 215798 T516 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/control-001/lib
   [junit4]   2> 215799 T516 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/control-001/lib).
   [junit4]   2> 215818 T516 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 215819 T516 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 215819 T516 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 215820 T516 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 215820 T516 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 215820 T516 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 215821 T516 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 215821 T516 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 215822 T516 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 215822 T516 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 215822 T516 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 215823 T516 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 215824 T516 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 215824 T516 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 215825 T516 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 215825 T516 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:13401/solr
   [junit4]   2> 215825 T516 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 215826 T516 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 215827 T516 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 215828 T539 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6a60d6e3 name:ZooKeeperConnection Watcher:127.0.0.1:13401 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 215829 T516 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 215830 T516 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 215832 T516 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 215834 T542 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4a5f4985 name:ZooKeeperConnection Watcher:127.0.0.1:13401/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 215834 T516 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 215836 T516 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 215838 T516 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 215840 T516 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 215842 T516 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 215844 T516 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 215846 T516 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 215847 T516 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 215848 T516 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 215850 T516 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54988_
   [junit4]   2> 215850 T516 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54988_
   [junit4]   2> 215852 T516 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 215853 T516 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 215855 T516 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 215856 T516 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:54988_
   [junit4]   2> 215857 T516 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 215858 T516 oasc.Overseer.start Overseer (id=93531074135916547-127.0.0.1:54988_-n_0000000000) starting
   [junit4]   2> 215860 T516 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 215865 T516 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 215866 T544 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 215866 T543 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 215866 T516 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 215870 T516 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/control-001/cores
   [junit4]   2> 215873 T516 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {loadOnStartup=true, name=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/control-001/cores/collection1/, coreNodeName=, schema=schema.xml, collection=control_collection, shard=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/control-001/cores/collection1, config=solrconfig.xml, transient=false, dataDir=data/}
   [junit4]   2> 215874 T516 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/control-001/cores/collection1/
   [junit4]   2> 215874 T516 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 215875 T546 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 215876 T546 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 215877 T546 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 215877 T542 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 215878 T543 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:54988_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54988",
   [junit4]   2> 	  "state":"down"} current state version: 0
   [junit4]   2> 215878 T543 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:54988_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54988",
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 215878 T543 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 215879 T543 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 215880 T542 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 216877 T546 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 216878 T546 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 216879 T546 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 216879 T546 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 216880 T546 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 216880 T546 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/control-001/cores/collection1/'
   [junit4]   2> 216904 T546 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 216906 T546 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 216920 T546 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 216944 T546 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 216997 T546 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 216999 T546 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 217012 T546 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 217480 T546 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 217499 T546 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 217503 T546 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 217519 T546 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 217524 T546 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 217535 T546 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 217536 T546 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 217536 T546 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 217537 T546 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 217537 T546 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 217538 T546 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 217538 T546 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 217538 T546 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 217539 T546 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/control-001/cores/collection1/data/
   [junit4]   2> 217539 T546 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@568258ac
   [junit4]   2> 217540 T546 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/control-001/cores/collection1/data
   [junit4]   2> 217541 T546 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/control-001/cores/collection1/data/index/
   [junit4]   2> 217541 T546 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 217541 T546 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/control-001/cores/collection1/data/index
   [junit4]   2> 217542 T546 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=32, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8335704942511237]
   [junit4]   2> 217543 T546 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@fe9be90 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3adde819),segFN=segments_1,generation=1}
   [junit4]   2> 217543 T546 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 217551 T546 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 217552 T546 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 217552 T546 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 217552 T546 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 217553 T546 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 217553 T546 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 217554 T546 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 217554 T546 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 217554 T546 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 217573 T546 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 217576 T546 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 217578 T546 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 217581 T546 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 217587 T546 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/properties,/config,/admin/segments,/update,/update/csv,/admin/file,/update/json,/schema,/replication,/update/json/docs,standard,/get,/admin/system,/admin/luke,/admin/threads,/admin/ping,/admin/logging,/admin/plugins,/admin/mbeans
   [junit4]   2> 217593 T546 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 217595 T546 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 217595 T546 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 217596 T546 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 217597 T546 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 217598 T546 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=41, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 217598 T546 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@fe9be90 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3adde819),segFN=segments_1,generation=1}
   [junit4]   2> 217599 T546 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 217599 T546 oass.SolrIndexSearcher.<init> Opening Searcher@14f9e396[collection1] main
   [junit4]   2> 217600 T546 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 217601 T546 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 217601 T546 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 217601 T546 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 217602 T546 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 217602 T546 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 217603 T546 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 217603 T546 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 217603 T546 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 217604 T546 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 217605 T547 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@14f9e396[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 217605 T546 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 217606 T550 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:54988 collection:control_collection shard:shard1
   [junit4]   2> 217606 T516 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 217607 T516 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 217607 T550 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 217611 T516 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 217611 T550 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 217612 T516 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 217613 T554 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@72f241c name:ZooKeeperConnection Watcher:127.0.0.1:13401/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 217613 T516 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 217614 T550 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 217613 T542 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 217614 T516 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 217614 T550 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 217615 T543 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 217615 T516 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> ASYNC  NEW_CORE C96 name=collection1 org.apache.solr.core.SolrCore@8799e3a url=http://127.0.0.1:54988/collection1 node=127.0.0.1:54988_ C96_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:54988_, core=collection1, base_url=http://127.0.0.1:54988, state=down}
   [junit4]   2> 217615 T550 C96 P54988 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:54988/collection1/
   [junit4]   2> 217616 T550 C96 P54988 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 217616 T550 C96 P54988 oasc.SyncStrategy.syncToMe http://127.0.0.1:54988/collection1/ has no replicas
   [junit4]   2> 217617 T554 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 217617 T542 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 217617 T550 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:54988/collection1/ shard1
   [junit4]   2> 217618 T550 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 217618 T516 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 217619 T516 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 217619 T516 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 217620 T516 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 217621 T557 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1a880662 name:ZooKeeperConnection Watcher:127.0.0.1:13401/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 217621 T516 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 217622 T542 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 217622 T516 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 217623 T543 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54988",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 217625 T543 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 217626 T543 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 217627 T542 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 217627 T554 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 217629 T542 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 217630 T543 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 217631 T542 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 217631 T554 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 217672 T550 oasc.ZkController.register We are http://127.0.0.1:54988/collection1/ and leader is http://127.0.0.1:54988/collection1/
   [junit4]   2> 217673 T550 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:54988
   [junit4]   2> 217673 T550 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 217673 T550 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 217673 T550 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 217674 T542 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 217675 T543 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:54988_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54988",
   [junit4]   2> 	  "state":"active"} current state version: 4
   [junit4]   2> 217676 T543 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:54988_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54988",
   [junit4]   2> 	  "state":"active"}
   [junit4]   2> 217780 T542 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 217780 T554 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 219265 T516 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/shard-1-001/cores/collection1
   [junit4]   2> 219268 T516 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/shard-1-001
   [junit4]   2> 219268 T516 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 219271 T516 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:61340
   [junit4]   2> 219272 T516 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/shard-1-001/cores, solrconfig=solrconfig.xml, hostPort=48874, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/tempDir-001/jetty1, hostContext=/}
   [junit4]   2> 219272 T516 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 219272 T516 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/shard-1-001/'
   [junit4]   2> 219315 T516 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/shard-1-001/solr.xml
   [junit4]   2> 219370 T516 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/shard-1-001/cores
   [junit4]   2> 219371 T516 oasc.CoreContainer.<init> New CoreContainer 521581779
   [junit4]   2> 219371 T516 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/shard-1-001/]
   [junit4]   2> 219371 T516 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/shard-1-001/lib
   [junit4]   2> 219372 T516 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/shard-1-001/lib).
   [junit4]   2> 219382 T516 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 219383 T516 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 219383 T516 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 219383 T516 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 219384 T516 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 219384 T516 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 219384 T516 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 219384 T516 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 219385 T516 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 219385 T516 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 219385 T516 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 219386 T516 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 219386 T516 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 219386 T516 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 219387 T516 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 219387 T516 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:13401/solr
   [junit4]   2> 219387 T516 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 219388 T516 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 219388 T516 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 219390 T569 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@117a5f43 name:ZooKeeperConnection Watcher:127.0.0.1:13401 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 219391 T516 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 219392 T516 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 219394 T516 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 219395 T572 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@11a670b5 name:ZooKeeperConnection Watcher:127.0.0.1:13401/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 219395 T516 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 219400 T516 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 220405 T516 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48874_
   [junit4]   2> 220406 T516 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:48874_
   [junit4]   2> 220409 T516 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 220411 T516 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/shard-1-001/cores
   [junit4]   2> 220414 T516 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {dataDir=data/, shard=, config=solrconfig.xml, schema=schema.xml, loadOnStartup=true, collection=collection1, transient=false, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/shard-1-001/cores/collection1/, coreNodeName=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/shard-1-001/cores/collection1, name=collection1}
   [junit4]   2> 220415 T516 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/shard-1-001/cores/collection1/
   [junit4]   2> 220415 T516 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 220418 T573 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 220418 T573 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 220419 T542 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 220420 T573 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 220420 T573 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 220420 T543 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:48874_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48874",
   [junit4]   2> 	  "state":"down"} current state version: 5
   [junit4]   2> 220420 T573 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 220421 T543 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:48874_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48874",
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 220421 T543 oasco.ReplicaMutator.updateState Collection already exists with numShards=3
   [junit4]   2> 220422 T543 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 220422 T573 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 220422 T573 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 220423 T543 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 220424 T572 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 220424 T572 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 221422 T573 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 221423 T573 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 221424 T573 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 221424 T573 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 221425 T573 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 221425 T573 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/shard-1-001/cores/collection1/'
   [junit4]   2> 221451 T573 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 221454 T573 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 221467 T573 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 221491 T573 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 221552 T573 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 221554 T573 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 221585 T573 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 221993 T573 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 222015 T573 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 222017 T573 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 222030 T573 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 222033 T573 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 222036 T573 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 222037 T573 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 222037 T573 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 222037 T573 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 222038 T573 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 222038 T573 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 222038 T573 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 222038 T573 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 222039 T573 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 222039 T573 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@568258ac
   [junit4]   2> 222040 T573 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/shard-1-001/cores/collection1/data
   [junit4]   2> 222040 T573 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 222040 T573 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 222041 T573 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 222041 T573 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=32, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8335704942511237]
   [junit4]   2> 222042 T573 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@10b5b606 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@9c14c7c),segFN=segments_1,generation=1}
   [junit4]   2> 222042 T573 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 222048 T573 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 222049 T573 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 222049 T573 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 222049 T573 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 222049 T573 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 222050 T573 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 222050 T573 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 222050 T573 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 222051 T573 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 222063 T573 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 222064 T573 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 222066 T573 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 222068 T573 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 222075 T573 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/properties,/config,/admin/segments,/update,/update/csv,/admin/file,/update/json,/schema,/replication,/update/json/docs,standard,/get,/admin/system,/admin/luke,/admin/threads,/admin/ping,/admin/logging,/admin/plugins,/admin/mbeans
   [junit4]   2> 222077 T573 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 222078 T573 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 222078 T573 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 222079 T573 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 222080 T573 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 222080 T573 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=41, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 222081 T573 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@10b5b606 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@9c14c7c),segFN=segments_1,generation=1}
   [junit4]   2> 222081 T573 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 222082 T573 oass.SolrIndexSearcher.<init> Opening Searcher@2c8de917[collection1] main
   [junit4]   2> 222082 T573 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 222083 T573 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 222083 T573 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 222083 T573 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 222083 T573 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 222084 T573 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 222084 T573 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 222084 T573 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 222085 T573 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 222085 T573 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 222086 T574 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2c8de917[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 222087 T573 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 222088 T577 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:48874 collection:collection1 shard:shard1
   [junit4]   2> 222088 T516 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 222089 T516 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 222089 T577 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 222093 T577 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 222095 T542 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 222095 T577 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 222095 T577 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C97 name=collection1 org.apache.solr.core.SolrCore@7ea187d3 url=http://127.0.0.1:48874/collection1 node=127.0.0.1:48874_ C97_STATE=coll:collection1 core:collection1 props:{node_name=127.0.0.1:48874_, core=collection1, base_url=http://127.0.0.1:48874, state=down}
   [junit4]   2> 222096 T577 C97 P48874 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:48874/collection1/
   [junit4]   2> 222096 T543 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 222096 T577 C97 P48874 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 222097 T577 C97 P48874 oasc.SyncStrategy.syncToMe http://127.0.0.1:48874/collection1/ has no replicas
   [junit4]   2> 222097 T577 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:48874/collection1/ shard1
   [junit4]   2> 222097 T577 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 222097 T543 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 222098 T572 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 222099 T572 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 222101 T543 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48874",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 222102 T543 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 222103 T572 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 222103 T572 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 222105 T542 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 222151 T577 oasc.ZkController.register We are http://127.0.0.1:48874/collection1/ and leader is http://127.0.0.1:48874/collection1/
   [junit4]   2> 222152 T577 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:48874
   [junit4]   2> 222152 T577 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 222152 T577 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 222153 T577 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 222154 T542 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 222155 T543 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:48874_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48874",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 222155 T543 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:48874_",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48874",
   [junit4]   2> 	  "state":"active"}
   [junit4]   2> 222156 T577 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 222157 T543 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 3
   [junit4]   2> 222157 T572 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 222158 T572 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 4 
   [junit4]   2> 223890 T516 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/shard-2-001/cores/collection1
   [junit4]   2> 223893 T516 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/shard-2-001
   [junit4]   2> 223894 T516 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 223896 T516 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:38963
   [junit4]   2> 223897 T516 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solrconfig=solrconfig.xml, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/shard-2-001/cores, hostContext=/, hostPort=22671, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/tempDir-001/jetty2}
   [junit4]   2> 223897 T516 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 223897 T516 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/shard-2-001/'
   [junit4]   2> 223946 T516 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/shard-2-001/solr.xml
   [junit4]   2> 224064 T516 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/shard-2-001/cores
   [junit4]   2> 224065 T516 oasc.CoreContainer.<init> New CoreContainer 643341901
   [junit4]   2> 224065 T516 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/shard-2-001/]
   [junit4]   2> 224066 T516 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/shard-2-001/lib
   [junit4]   2> 224067 T516 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest AE25518C57560B2C-001/shard-2-001/lib).
   [junit4]   2> 224081 T516 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 224082 T516 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 224083 T516 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 224083 T516 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 224083 T516 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 224084 T516 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 224084 T516 oashc.HttpShardHandlerF

[...truncated too long message...]

va:135)
   [junit4]   2> 		... 11 more
   [junit4]   2> 	
   [junit4]   2> 1530338 T3678 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> 1530462 T3678 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:45458 45458
   [junit4]   2> 1531669 T3679 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:45458 45458
   [junit4]   2> 1531670 T3679 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/aliases.json
   [junit4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 		4	/solr/collections/collection1/state.json
   [junit4]   2> 		3	/solr/collections/c8n_1x3_commits/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent data watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		2	/solr/collections/c8n_1x3_commits/leader_elect/shard1/election/93531155668533261-core_node2-n_0000000000
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 		5	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1531670 T3678 oasc.SocketProxy.close WARN Closing 11 connections to: http://127.0.0.1:45475/
   [junit4]   2> 1531671 T3678 oasc.SocketProxy.close WARN Closing 3 connections to: http://127.0.0.1:19002/
   [junit4]   2> 1531671 T3678 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:45464/
   [junit4]   2> 1531671 T3678 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:17611/
   [junit4]   2> 1531672 T3678 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:47637/
   [junit4]   2> 1531672 T3678 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:45475/
   [junit4]   2> 1531672 T3678 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:19002/
   [junit4]   2> 1531672 T3678 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:45464/
   [junit4]   2> 1531672 T3678 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:17611/
   [junit4]   2> 1531673 T3678 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:47637/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderInitiatedRecoveryOnCommitTest -Dtests.method=test -Dtests.seed=AE25518C57560B2C -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ms_MY -Dtests.timezone=Asia/Hebron -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   74.5s J1 | LeaderInitiatedRecoveryOnCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:17611/c8n_1x3_commits_shard1_replica2
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([AE25518C57560B2C:26716E56F9AA66D4]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
   [junit4]    > 	... 48 more
   [junit4]   2> 1531693 T3678 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest AE25518C57560B2C-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50), sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {}, locale=ms_MY, timezone=Asia/Hebron
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=91372760,total=319815680
   [junit4]   2> NOTE: All tests run in this JVM: [CoreMergeIndexesAdminHandlerTest, TestJsonFacets, HighlighterTest, TestArbitraryIndexDir, TestSweetSpotSimilarityFactory, TestRawResponseWriter, OverseerCollectionProcessorTest, TestSerializedLuceneMatchVersion, TestStandardQParsers, EchoParamsTest, TestSimpleQParserPlugin, ExitableDirectoryReaderTest, StatelessScriptUpdateProcessorFactoryTest, TestCoreDiscovery, TestLRUCache, TestTrie, DocValuesTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, SpellingQueryConverterTest, SolrCoreTest, UniqFieldsUpdateProcessorFactoryTest, SolrXmlInZkTest, LoggingHandlerTest, UpdateRequestProcessorFactoryTest, TestCursorMarkWithoutUniqueKey, TestCSVLoader, ClusterStateUpdateTest, TestManagedSynonymFilterFactory, TestJmxIntegration, ScriptEngineTest, OverseerStatusTest, DocumentAnalysisRequestHandlerTest, OverseerRolesTest, CursorPagingTest, TestTolerantSearch, AnalysisAfterCoreReloadTest, TestFastLRUCache, NumericFieldsTest, RegexBoostProcessorTest, HdfsRecoveryZkTest, TestSolrDeletionPolicy2, CircularListTest, TestSolrQueryParserResource, MergeStrategyTest, TestDistribDocBasedVersion, ChangedSchemaMergeTest, PolyFieldTest, TestManagedResourceStorage, TestTrackingShardHandlerFactory, HdfsCollectionsAPIDistributedZkTest, FullSolrCloudDistribCmdsTest, BadComponentTest, TestStressRecovery, DefaultValueUpdateProcessorTest, TestDistributedGrouping, TestRequestStatusCollectionAPI, SpatialHeatmapFacetsTest, ZkCLITest, SampleTest, TestCollectionAPI, HttpPartitionTest, TestCloudSchemaless, LeaderInitiatedRecoveryOnCommitTest]
   [junit4] Completed on J1 in 74.50s, 1 test, 1 error <<< FAILURES!

[...truncated 695 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 475 suites, 1892 tests, 3 errors, 62 ignored (22 assumptions)

Total time: 55 minutes 21 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 191911175 bytes
Compression is 0.0%
Took 39 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2823 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2823/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:50984/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:50984/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([1976AF336B21DF41:912290E9C5DDB2B9]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:30384/udn/r/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:30384/udn/r/collection1
	at __randomizedtesting.SeedInfo.seed([1976AF336B21DF41:912290E9C5DDB2B9]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:30384/udn/r/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:12913/_/r/repfacttest_c8n_1x3_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:12913/_/r/repfacttest_c8n_1x3_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([1976AF336B21DF41:912290E9C5DDB2B9]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:12913/_/r/repfacttest_c8n_1x3_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 53 more




Build Log:
[...truncated 9406 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/init-core-data-001
   [junit4]   2> 261792 T1185 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /udn/r
   [junit4]   2> 261799 T1185 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 261800 T1186 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 261900 T1185 oasc.ZkTestServer.run start zk server on port:30372
   [junit4]   2> 261901 T1185 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 261902 T1185 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 261907 T1193 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@87aefa name:ZooKeeperConnection Watcher:127.0.0.1:30372 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 261907 T1185 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 261908 T1185 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 261908 T1185 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 261910 T1185 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 261911 T1185 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 261913 T1196 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3a8923ff name:ZooKeeperConnection Watcher:127.0.0.1:30372/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 261913 T1185 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 261914 T1185 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 261914 T1185 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 261916 T1185 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 261918 T1185 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 261919 T1185 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 261921 T1185 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 261922 T1185 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 261924 T1185 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 261925 T1185 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 261927 T1185 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 261928 T1185 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 261930 T1185 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 261930 T1185 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 261932 T1185 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 261933 T1185 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 261934 T1185 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 261935 T1185 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 261936 T1185 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 261937 T1185 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 261939 T1185 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 261939 T1185 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 261941 T1185 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 261942 T1185 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 261943 T1185 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 261944 T1185 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 261946 T1185 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 261946 T1185 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 263281 T1185 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/control-001/cores/collection1
   [junit4]   2> 263284 T1185 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 263287 T1185 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:30379
   [junit4]   2> 263288 T1185 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/udn/r, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/control-001/cores, hostPort=30378, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/tempDir-001/control/data}
   [junit4]   2> 263288 T1185 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 263288 T1185 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/control-001/'
   [junit4]   2> 263317 T1185 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/control-001/solr.xml
   [junit4]   2> 263352 T1185 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/control-001/cores
   [junit4]   2> 263352 T1185 oasc.CoreContainer.<init> New CoreContainer 1962020692
   [junit4]   2> 263352 T1185 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/control-001/]
   [junit4]   2> 263353 T1185 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/control-001/lib
   [junit4]   2> 263353 T1185 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/control-001/lib).
   [junit4]   2> 263362 T1185 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 263362 T1185 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 263362 T1185 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 263363 T1185 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 263363 T1185 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 263363 T1185 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 263363 T1185 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 263364 T1185 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 263364 T1185 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 263364 T1185 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 263364 T1185 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 263365 T1185 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 263365 T1185 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 263366 T1185 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 263366 T1185 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 263366 T1185 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:30372/solr
   [junit4]   2> 263366 T1185 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 263367 T1185 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 263367 T1185 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 263369 T1208 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2e4785da name:ZooKeeperConnection Watcher:127.0.0.1:30372 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 263369 T1185 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 263370 T1185 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 263371 T1185 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 263372 T1211 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6c37407b name:ZooKeeperConnection Watcher:127.0.0.1:30372/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 263372 T1185 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 263373 T1185 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 263375 T1185 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 263376 T1185 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 263377 T1185 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 263379 T1185 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 263380 T1185 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 263381 T1185 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 263382 T1185 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 263383 T1185 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:30378_udn%2Fr
   [junit4]   2> 263383 T1185 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:30378_udn%2Fr
   [junit4]   2> 263384 T1185 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 263385 T1185 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 263386 T1185 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 263387 T1185 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:30378_udn%2Fr
   [junit4]   2> 263388 T1185 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 263389 T1185 oasc.Overseer.start Overseer (id=93530126124122115-127.0.0.1:30378_udn%2Fr-n_0000000000) starting
   [junit4]   2> 263390 T1185 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 263394 T1185 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 263394 T1213 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 263394 T1212 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 263394 T1185 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 263397 T1185 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/control-001/cores
   [junit4]   2> 263399 T1185 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {schema=schema.xml, loadOnStartup=true, transient=false, coreNodeName=, shard=, config=solrconfig.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/control-001/cores/collection1/, collection=control_collection, dataDir=data/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/control-001/cores/collection1, name=collection1}
   [junit4]   2> 263399 T1185 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/control-001/cores/collection1/
   [junit4]   2> 263399 T1185 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 263400 T1215 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 263400 T1215 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 263401 T1215 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 263401 T1211 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 263402 T1212 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:30378_udn%2Fr",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:30378/udn/r",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1"} current state version: 0
   [junit4]   2> 263402 T1212 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:30378_udn%2Fr",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:30378/udn/r",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 263403 T1212 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 263403 T1212 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 263404 T1211 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 264401 T1215 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 264402 T1215 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 264402 T1215 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 264403 T1215 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 264403 T1215 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 264403 T1215 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/control-001/cores/collection1/'
   [junit4]   2> 264416 T1215 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 264418 T1215 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 264426 T1215 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 264450 T1215 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 264485 T1215 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 264486 T1215 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 264498 T1215 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 264867 T1215 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 264882 T1215 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 264885 T1215 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 264896 T1215 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 264900 T1215 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 264904 T1215 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 264905 T1215 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 264905 T1215 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 264906 T1215 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 264906 T1215 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 264907 T1215 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 264907 T1215 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 264907 T1215 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 264908 T1215 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/control-001/cores/collection1/data/
   [junit4]   2> 264908 T1215 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2813f8ee
   [junit4]   2> 264909 T1215 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/control-001/cores/collection1/data
   [junit4]   2> 264909 T1215 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/control-001/cores/collection1/data/index/
   [junit4]   2> 264910 T1215 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 264910 T1215 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/control-001/cores/collection1/data/index
   [junit4]   2> 264911 T1215 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1143113919, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 264912 T1215 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@793630c7 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@54bc96fc),segFN=segments_1,generation=1}
   [junit4]   2> 264912 T1215 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 264919 T1215 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 264919 T1215 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 264920 T1215 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 264920 T1215 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 264920 T1215 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 264921 T1215 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 264921 T1215 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 264922 T1215 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 264922 T1215 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 264942 T1215 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 264945 T1215 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 264949 T1215 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 264952 T1215 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 264959 T1215 oasc.RequestHandlers.initHandlersFromConfig Registered paths: standard,/admin/logging,/update/json,/replication,/update,/admin/file,/admin/properties,/admin/mbeans,/admin/luke,/admin/threads,/admin/plugins,/config,/update/csv,/get,/admin/segments,/schema,/admin/ping,/admin/system,/update/json/docs
   [junit4]   2> 264961 T1215 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 264963 T1215 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 264963 T1215 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 264964 T1215 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 264965 T1215 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 264966 T1215 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=14, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.565266027770279]
   [junit4]   2> 264967 T1215 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@793630c7 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@54bc96fc),segFN=segments_1,generation=1}
   [junit4]   2> 264967 T1215 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 264968 T1215 oass.SolrIndexSearcher.<init> Opening Searcher@3a6161e2[collection1] main
   [junit4]   2> 264968 T1215 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 264969 T1215 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 264969 T1215 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 264970 T1215 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 264970 T1215 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 264971 T1215 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 264971 T1215 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 264972 T1215 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 264972 T1215 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 264972 T1215 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 264973 T1216 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3a6161e2[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 264974 T1215 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 264975 T1219 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:30378/udn/r collection:control_collection shard:shard1
   [junit4]   2> 264975 T1185 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 264975 T1185 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 264975 T1219 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 264978 T1185 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 264979 T1185 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 264979 T1219 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 264980 T1223 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7620d6ee name:ZooKeeperConnection Watcher:127.0.0.1:30372/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 264980 T1185 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 264981 T1219 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 264981 T1211 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 264981 T1185 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 264981 T1219 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 264982 T1212 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 264982 T1185 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> ASYNC  NEW_CORE C128 name=collection1 org.apache.solr.core.SolrCore@c5dc81a url=http://127.0.0.1:30378/udn/r/collection1 node=127.0.0.1:30378_udn%2Fr C128_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:30378_udn%2Fr, base_url=http://127.0.0.1:30378/udn/r, state=down, core=collection1}
   [junit4]   2> 264982 T1219 C128 P30378 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:30378/udn/r/collection1/
   [junit4]   2> 264983 T1219 C128 P30378 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 264983 T1223 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 264983 T1211 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 264983 T1219 C128 P30378 oasc.SyncStrategy.syncToMe http://127.0.0.1:30378/udn/r/collection1/ has no replicas
   [junit4]   2> 264984 T1219 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:30378/udn/r/collection1/ shard1
   [junit4]   2> 264985 T1219 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 264985 T1185 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 264987 T1211 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 264988 T1212 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:30378/udn/r",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 265090 T1211 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 265090 T1223 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 265140 T1219 oasc.ZkController.register We are http://127.0.0.1:30378/udn/r/collection1/ and leader is http://127.0.0.1:30378/udn/r/collection1/
   [junit4]   2> 265141 T1219 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:30378/udn/r
   [junit4]   2> 265141 T1219 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 265141 T1219 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 265142 T1219 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 265143 T1211 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 265143 T1212 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:30378_udn%2Fr",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:30378/udn/r",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1"} current state version: 3
   [junit4]   2> 265144 T1212 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:30378_udn%2Fr",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:30378/udn/r",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 265247 T1211 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 265247 T1223 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 266296 T1185 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/shard-1-001/cores/collection1
   [junit4]   2> 266298 T1185 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/shard-1-001
   [junit4]   2> 266299 T1185 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 266301 T1185 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:30385
   [junit4]   2> 266301 T1185 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/shard-1-001/cores, solrconfig=solrconfig.xml, hostPort=30384, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/tempDir-001/jetty1, hostContext=/udn/r}
   [junit4]   2> 266302 T1185 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 266302 T1185 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/shard-1-001/'
   [junit4]   2> 266335 T1185 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/shard-1-001/solr.xml
   [junit4]   2> 266374 T1185 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/shard-1-001/cores
   [junit4]   2> 266375 T1185 oasc.CoreContainer.<init> New CoreContainer 686336082
   [junit4]   2> 266375 T1185 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/shard-1-001/]
   [junit4]   2> 266375 T1185 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/shard-1-001/lib
   [junit4]   2> 266376 T1185 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/shard-1-001/lib).
   [junit4]   2> 266386 T1185 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 266386 T1185 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 266386 T1185 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 266387 T1185 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 266387 T1185 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 266387 T1185 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 266387 T1185 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 266388 T1185 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 266388 T1185 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 266388 T1185 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 266388 T1185 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 266389 T1185 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 266389 T1185 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 266390 T1185 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 266390 T1185 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 266390 T1185 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:30372/solr
   [junit4]   2> 266390 T1185 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 266391 T1185 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 266391 T1185 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 266393 T1235 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@f5217f2 name:ZooKeeperConnection Watcher:127.0.0.1:30372 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 266393 T1185 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 266394 T1185 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 266395 T1185 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 266396 T1238 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2394c1f9 name:ZooKeeperConnection Watcher:127.0.0.1:30372/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 266397 T1185 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 266400 T1185 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 267403 T1185 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:30384_udn%2Fr
   [junit4]   2> 267404 T1185 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:30384_udn%2Fr
   [junit4]   2> 267407 T1185 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 267409 T1185 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/shard-1-001/cores
   [junit4]   2> 267410 T1185 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {shard=, loadOnStartup=true, collection=collection1, dataDir=data/, config=solrconfig.xml, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/shard-1-001/cores/collection1, name=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/shard-1-001/cores/collection1/, transient=false, coreNodeName=}
   [junit4]   2> 267411 T1185 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/shard-1-001/cores/collection1/
   [junit4]   2> 267411 T1185 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 267412 T1239 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 267412 T1239 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 267413 T1239 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 267413 T1211 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 267414 T1212 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:30384_udn%2Fr",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:30384/udn/r",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1"} current state version: 4
   [junit4]   2> 267414 T1212 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:30384_udn%2Fr",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:30384/udn/r",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 267414 T1212 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 267414 T1212 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 267415 T1223 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 267415 T1238 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 267415 T1211 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 268414 T1239 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 268415 T1239 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 268415 T1239 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 268416 T1239 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 268416 T1239 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 268416 T1239 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/shard-1-001/cores/collection1/'
   [junit4]   2> 268428 T1239 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 268430 T1239 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 268437 T1239 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 268452 T1239 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 268484 T1239 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 268486 T1239 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 268493 T1239 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 268748 T1239 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 268758 T1239 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 268760 T1239 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 268769 T1239 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 268772 T1239 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 268775 T1239 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 268776 T1239 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 268776 T1239 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 268776 T1239 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 268777 T1239 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 268777 T1239 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 268777 T1239 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 268778 T1239 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 268778 T1239 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 268778 T1239 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2813f8ee
   [junit4]   2> 268779 T1239 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/shard-1-001/cores/collection1/data
   [junit4]   2> 268779 T1239 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 268779 T1239 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 268780 T1239 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 268780 T1239 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1143113919, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 268781 T1239 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2e354a96 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2cb3d58d),segFN=segments_1,generation=1}
   [junit4]   2> 268781 T1239 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 268787 T1239 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 268787 T1239 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 268787 T1239 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 268788 T1239 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 268788 T1239 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 268788 T1239 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 268788 T1239 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 268789 T1239 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 268789 T1239 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 268801 T1239 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 268803 T1239 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 268804 T1239 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 268806 T1239 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 268812 T1239 oasc.RequestHandlers.initHandlersFromConfig Registered paths: standard,/admin/logging,/update/json,/replication,/update,/admin/file,/admin/properties,/admin/mbeans,/admin/luke,/admin/threads,/admin/plugins,/config,/update/csv,/get,/admin/segments,/schema,/admin/ping,/admin/system,/update/json/docs
   [junit4]   2> 268813 T1239 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 268815 T1239 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 268815 T1239 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 268816 T1239 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 268816 T1239 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 268817 T1239 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=14, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.565266027770279]
   [junit4]   2> 268817 T1239 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2e354a96 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2cb3d58d),segFN=segments_1,generation=1}
   [junit4]   2> 268817 T1239 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 268818 T1239 oass.SolrIndexSearcher.<init> Opening Searcher@3e4b2d80[collection1] main
   [junit4]   2> 268818 T1239 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 268819 T1239 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 268819 T1239 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 268820 T1239 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 268820 T1239 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 268820 T1239 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 268821 T1239 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 268821 T1239 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 268821 T1239 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 268821 T1239 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 268822 T1240 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3e4b2d80[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 268823 T1239 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 268824 T1243 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:30384/udn/r collection:collection1 shard:shard1
   [junit4]   2> 268824 T1185 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2
   [junit4]   2> 268824 T1185 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 268825 T1243 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 268829 T1243 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 268830 T1211 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 268830 T1243 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 268831 T1243 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C129 name=collection1 org.apache.solr.core.SolrCore@24564d4a url=http://127.0.0.1:30384/udn/r/collection1 node=127.0.0.1:30384_udn%2Fr C129_STATE=coll:collection1 core:collection1 props:{node_name=127.0.0.1:30384_udn%2Fr, base_url=http://127.0.0.1:30384/udn/r, state=down, core=collection1}
   [junit4]   2> 268831 T1243 C129 P30384 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:30384/udn/r/collection1/
   [junit4]   2> 268831 T1212 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 268831 T1243 C129 P30384 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 268832 T1243 C129 P30384 oasc.SyncStrategy.syncToMe http://127.0.0.1:30384/udn/r/collection1/ has no replicas
   [junit4]   2> 268832 T1243 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:30384/udn/r/collection1/ shard1
   [junit4]   2> 268832 T1243 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 268835 T1211 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 268836 T1212 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:30384/udn/r",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 268939 T1211 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 268939 T1223 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 268939 T1238 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 268989 T1243 oasc.ZkController.register We are http://127.0.0.1:30384/udn/r/collection1/ and leader is http://127.0.0.1:30384/udn/r/collection1/
   [junit4]   2> 268990 T1243 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:30384/udn/r
   [junit4]   2> 268990 T1243 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 268990 T1243 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 268990 T1243 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 268992 T1211 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 268992 T1212 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:30384_udn%2Fr",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:30384/udn/r",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1"} current state version: 6
   [junit4]   2> 268993 T1212 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:30384_udn%2Fr",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:30384/udn/r",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 269097 T1223 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 269097 T1211 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 269097 T1238 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 270238 T1185 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/shard-2-001/cores/collection1
   [junit4]   2> 270240 T1185 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/shard-2-001
   [junit4]   2> 270241 T1185 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 270243 T1185 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:48250
   [junit4]   2> 270243 T1185 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/tempDir-001/jetty2, hostContext=/udn/r, solrconfig=solrconfig.xml, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/shard-2-001/cores, hostPort=55191}
   [junit4]   2> 270244 T1185 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 270244 T1185 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/shard-2-001/'
   [junit4]   2> 270274 T1185 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/shard-2-001/solr.xml
   [junit4]   2> 270309 T1185 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/shard-2-001/cores
   [junit4]   2> 270309 T1185 oasc.CoreContainer.<init> New CoreContainer 1195813640
   [junit4]   2> 270310 T1185 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/shard-2-001/]
   [junit4]   2> 270310 T1185 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/shard-2-001/lib
   [junit4]   2> 270311 T1185 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 1976AF336B21DF41-001/shard-2-001/lib).
   [junit4]   2> 270319 T1185 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 270320 T1185 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 270320 T1185 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 270320 T1185 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 270320 T1185 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 270331 T1185 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 270332 T1185 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 270332 T1185 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 270332 T1185 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 270332 T1185 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 270333 T1185 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 270333 T1185 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 270333 T1185 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 270334 T1185 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 270334 T1185 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 270334 T1185 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:30372/solr
   [junit4]   2> 270335 T1185 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 270335 T1185 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 270335 T1185 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 270337 T1256 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5acabab name:ZooKeeperConnection Watcher:127.0.0.1:30372 got event WatchedEvent state:S

[...truncated too long message...]

nt create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		5	/solr/clusterstate.json
   [junit4]   2> 		5	/solr/aliases.json
   [junit4]   2> 		4	/solr/configs/conf1
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		5	/solr/live_nodes
   [junit4]   2> 		4	/solr/overseer/collection-queue-work
   [junit4]   2> 		4	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1465320 T3204 oasc.SocketProxy.close WARN Closing 3 connections to: http://127.0.0.1:12952/_/r
   [junit4]   2> 1465320 T3204 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:12931/_/r
   [junit4]   2> 1465321 T3204 oasc.SocketProxy.close WARN Closing 9 connections to: http://127.0.0.1:12913/_/r
   [junit4]   2> 1465321 T3204 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:12919/_/r
   [junit4]   2> 1465321 T3204 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 1465322 T3204 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:12952/_/r
   [junit4]   2> 1465322 T3204 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:12931/_/r
   [junit4]   2> 1465322 T3204 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:12913/_/r
   [junit4]   2> 1465322 T3204 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:12919/_/r
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=1976AF336B21DF41 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ga -Dtests.timezone=Indian/Reunion -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   76.5s J1 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:12913/_/r/repfacttest_c8n_1x3_shard1_replica3
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([1976AF336B21DF41:912290E9C5DDB2B9]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:12913/_/r/repfacttest_c8n_1x3_shard1_replica3
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 48 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
   [junit4]    > 	... 53 more
   [junit4]   2> 1465336 T3204 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 1976AF336B21DF41-001
   [junit4]   2> 76541 T3203 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Lucene50, sim=DefaultSimilarity, locale=ga, timezone=Indian/Reunion
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=115615368,total=324009984
   [junit4]   2> NOTE: All tests run in this JVM: [SynonymTokenizerTest, TimeZoneUtilsTest, TestCollapseQParserPlugin, DistributedIntervalFacetingTest, TestFieldCollectionResource, EnumFieldTest, PeerSyncTest, CursorMarkTest, TestWordDelimiterFilterFactory, HdfsDirectoryFactoryTest, TestSimpleQParserPlugin, SolrCmdDistributorTest, TestManagedStopFilterFactory, TestSolrIndexConfig, TestPerFieldSimilarity, TestDefaultSearchFieldResource, SegmentsInfoRequestHandlerTest, TestDistributedGrouping, TestPhraseSuggestions, HdfsWriteToMultipleCollectionsTest, TestFreeTextSuggestions, SolrCoreCheckLockOnStartupTest, LeaderFailoverAfterPartitionTest, XsltUpdateRequestHandlerTest, HdfsChaosMonkeySafeLeaderTest, CoreAdminHandlerTest, TestExtendedDismaxParser, DistanceFunctionTest, TestRequestStatusCollectionAPI, BadIndexSchemaTest, NotRequiredUniqueKeyTest, TestManagedSchemaDynamicFieldResource, TestReload, ChaosMonkeyNothingIsSafeTest, TestSolrDynamicMBean, TestConfigReload, TestBlendedInfixSuggestions, IndexSchemaTest, SolrXmlInZkTest, TestTrie, RankQueryTest, TestSolrQueryParser, PolyFieldTest, TestPartialUpdateDeduplication, TestAtomicUpdateErrorCases, DefaultValueUpdateProcessorTest, TestPseudoReturnFields, CloudMLTQParserTest, TestFieldTypeResource, SolrRequestParserTest, IndexSchemaRuntimeFieldTest, RegexBoostProcessorTest, OverseerStatusTest, TriLevelCompositeIdRoutingTest, TestSolr4Spatial, LeaderInitiatedRecoveryOnCommitTest, TermVectorComponentDistributedTest, ShardSplitTest, TestCloudSchemaless, ReplicationFactorTest]
   [junit4] Completed on J1 in 76.83s, 1 test, 1 error <<< FAILURES!

[...truncated 710 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 475 suites, 1892 tests, 3 errors, 62 ignored (22 assumptions)

Total time: 51 minutes 34 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 206296935 bytes
Compression is 0.0%
Took 38 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2822 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2822/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:58686/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:58686/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([DBB40DFD5B28D458:53E03227F5D4B9A0]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([DBB40DFD5B28D458:53E03227F5D4B9A0]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([DBB40DFD5B28D458:53E03227F5D4B9A0]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9754 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/init-core-data-001
   [junit4]   2> 911386 T3842 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 911396 T3842 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 911397 T3843 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 911497 T3842 oasc.ZkTestServer.run start zk server on port:58677
   [junit4]   2> 911498 T3842 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 911499 T3842 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 911504 T3850 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4b6c3adb name:ZooKeeperConnection Watcher:127.0.0.1:58677 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 911504 T3842 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 911505 T3842 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 911505 T3842 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 911509 T3842 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 911510 T3842 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 911512 T3853 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1ed05dd0 name:ZooKeeperConnection Watcher:127.0.0.1:58677/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 911512 T3842 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 911512 T3842 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 911513 T3842 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 911516 T3842 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 911518 T3842 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 911519 T3842 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 911521 T3842 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 911522 T3842 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 911525 T3842 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 911526 T3842 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 911528 T3842 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 911529 T3842 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 911531 T3842 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 911531 T3842 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 911533 T3842 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 911534 T3842 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 911536 T3842 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 911537 T3842 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 911538 T3842 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 911539 T3842 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 911542 T3842 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 911543 T3842 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 911545 T3842 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 911546 T3842 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 911548 T3842 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 911548 T3842 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 911550 T3842 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 911551 T3842 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 913208 T3842 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/control-001/cores/collection1
   [junit4]   2> 913212 T3842 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 913218 T3842 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:58681
   [junit4]   2> 913218 T3842 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, hostPort=58680, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/control-001/cores}
   [junit4]   2> 913219 T3842 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 913220 T3842 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/control-001/'
   [junit4]   2> 913280 T3842 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/control-001/solr.xml
   [junit4]   2> 913339 T3842 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/control-001/cores
   [junit4]   2> 913339 T3842 oasc.CoreContainer.<init> New CoreContainer 439335140
   [junit4]   2> 913339 T3842 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/control-001/]
   [junit4]   2> 913340 T3842 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/control-001/lib
   [junit4]   2> 913341 T3842 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/control-001/lib).
   [junit4]   2> 913355 T3842 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 913355 T3842 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 913355 T3842 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 913356 T3842 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 913356 T3842 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 913356 T3842 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 913357 T3842 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 913357 T3842 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 913357 T3842 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 913358 T3842 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 913358 T3842 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 913359 T3842 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 913359 T3842 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 913359 T3842 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 913360 T3842 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 913360 T3842 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:58677/solr
   [junit4]   2> 913360 T3842 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 913361 T3842 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 913362 T3842 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 913365 T3865 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1888b37e name:ZooKeeperConnection Watcher:127.0.0.1:58677 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 913366 T3842 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 913366 T3842 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 913368 T3842 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 913370 T3868 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4c3c6254 name:ZooKeeperConnection Watcher:127.0.0.1:58677/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 913370 T3842 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 913371 T3842 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 913373 T3842 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 913375 T3842 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 913377 T3842 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 913379 T3842 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 913381 T3842 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 913383 T3842 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 913384 T3842 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 913385 T3842 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:58680_
   [junit4]   2> 913386 T3842 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:58680_
   [junit4]   2> 913388 T3842 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 913389 T3842 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 913390 T3842 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 913392 T3842 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:58680_
   [junit4]   2> 913392 T3842 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 913394 T3842 oasc.Overseer.start Overseer (id=93529221862195203-127.0.0.1:58680_-n_0000000000) starting
   [junit4]   2> 913396 T3842 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 913401 T3842 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 913402 T3870 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 913402 T3842 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 913402 T3869 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 913407 T3842 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/control-001/cores
   [junit4]   2> 913409 T3842 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {coreNodeName=, name=collection1, loadOnStartup=true, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/control-001/cores/collection1/, collection=control_collection, schema=schema.xml, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/control-001/cores/collection1, transient=false, dataDir=data/, shard=}
   [junit4]   2> 913410 T3842 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/control-001/cores/collection1/
   [junit4]   2> 913410 T3842 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 913411 T3872 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 913411 T3872 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 913412 T3868 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 913412 T3872 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 913413 T3869 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58680",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:58680_",
   [junit4]   2> 	  "state":"down"} current state version: 0
   [junit4]   2> 913414 T3869 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58680",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:58680_",
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 913414 T3869 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 913415 T3869 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 913416 T3868 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 914413 T3872 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 914414 T3872 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 914414 T3872 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 914415 T3872 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 914415 T3872 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 914416 T3872 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/control-001/cores/collection1/'
   [junit4]   2> 914430 T3872 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 914432 T3872 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 914444 T3872 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 914469 T3872 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 914520 T3872 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 914521 T3872 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 914532 T3872 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 914958 T3872 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 914975 T3872 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 914978 T3872 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 914989 T3872 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 914995 T3872 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 915000 T3872 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 915001 T3872 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 915001 T3872 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 915002 T3872 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 915002 T3872 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 915003 T3872 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 915003 T3872 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 915003 T3872 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 915004 T3872 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/control-001/cores/collection1/data/
   [junit4]   2> 915004 T3872 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@638c7ceb
   [junit4]   2> 915005 T3872 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/control-001/cores/collection1/data
   [junit4]   2> 915005 T3872 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/control-001/cores/collection1/data/index/
   [junit4]   2> 915005 T3872 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 915006 T3872 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/control-001/cores/collection1/data/index
   [junit4]   2> 915006 T3872 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1062300316, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 915007 T3872 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@12433b49 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7be93efc),segFN=segments_1,generation=1}
   [junit4]   2> 915007 T3872 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 915013 T3872 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 915013 T3872 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 915014 T3872 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 915014 T3872 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 915014 T3872 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 915014 T3872 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 915015 T3872 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 915015 T3872 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 915015 T3872 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 915037 T3872 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 915040 T3872 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 915042 T3872 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 915044 T3872 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 915049 T3872 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/plugins,/schema,/config,/update/csv,/admin/system,/admin/segments,/admin/ping,/admin/luke,/admin/threads,/admin/logging,/admin/properties,standard,/update/json,/get,/replication,/update,/update/json/docs,/admin/file,/admin/mbeans
   [junit4]   2> 915052 T3872 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 915053 T3872 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 915053 T3872 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 915055 T3872 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 915055 T3872 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 915056 T3872 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=37, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 915057 T3872 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@12433b49 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7be93efc),segFN=segments_1,generation=1}
   [junit4]   2> 915057 T3872 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 915058 T3872 oass.SolrIndexSearcher.<init> Opening Searcher@55adc5b4[collection1] main
   [junit4]   2> 915058 T3872 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 915059 T3872 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 915059 T3872 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 915060 T3872 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 915060 T3872 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 915060 T3872 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 915061 T3872 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 915061 T3872 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 915061 T3872 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 915061 T3872 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 915062 T3873 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@55adc5b4[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 915063 T3872 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 915064 T3876 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:58680 collection:control_collection shard:shard1
   [junit4]   2> 915064 T3842 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 915064 T3842 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 915064 T3876 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 915068 T3842 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 915069 T3842 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 915069 T3876 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 915070 T3880 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1138d28 name:ZooKeeperConnection Watcher:127.0.0.1:58677/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 915071 T3842 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 915071 T3842 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 915071 T3842 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 915071 T3868 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 915072 T3876 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 915072 T3876 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C530 name=collection1 org.apache.solr.core.SolrCore@5d970970 url=http://127.0.0.1:58680/collection1 node=127.0.0.1:58680_ C530_STATE=coll:control_collection core:collection1 props:{core=collection1, base_url=http://127.0.0.1:58680, node_name=127.0.0.1:58680_, state=down}
   [junit4]   2> 915072 T3876 C530 P58680 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:58680/collection1/
   [junit4]   2> 915072 T3869 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 915073 T3876 C530 P58680 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 915073 T3876 C530 P58680 oasc.SyncStrategy.syncToMe http://127.0.0.1:58680/collection1/ has no replicas
   [junit4]   2> 915073 T3876 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:58680/collection1/ shard1
   [junit4]   2> 915074 T3876 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 915074 T3842 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 915074 T3868 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 915074 T3880 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 915075 T3842 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 915076 T3842 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 915077 T3842 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 915079 T3883 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@743f8b1f name:ZooKeeperConnection Watcher:127.0.0.1:58677/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 915079 T3842 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 915079 T3842 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 915080 T3868 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 915081 T3869 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58680",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 915083 T3869 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 915083 T3869 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 915084 T3868 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 915084 T3880 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 915086 T3868 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 915087 T3869 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 915088 T3868 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 915088 T3880 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 915137 T3876 oasc.ZkController.register We are http://127.0.0.1:58680/collection1/ and leader is http://127.0.0.1:58680/collection1/
   [junit4]   2> 915138 T3876 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:58680
   [junit4]   2> 915138 T3876 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 915138 T3876 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 915138 T3876 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 915139 T3868 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 915140 T3869 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58680",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:58680_",
   [junit4]   2> 	  "state":"active"} current state version: 4
   [junit4]   2> 915141 T3869 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58680",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:58680_",
   [junit4]   2> 	  "state":"active"}
   [junit4]   2> 915245 T3868 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 915245 T3880 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 916735 T3842 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/shard-1-001/cores/collection1
   [junit4]   2> 916738 T3842 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/shard-1-001
   [junit4]   2> 916739 T3842 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 916743 T3842 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:58687
   [junit4]   2> 916743 T3842 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=58686, solrconfig=solrconfig.xml, hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/shard-1-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/tempDir-001/jetty1}
   [junit4]   2> 916744 T3842 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 916744 T3842 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/shard-1-001/'
   [junit4]   2> 916799 T3842 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/shard-1-001/solr.xml
   [junit4]   2> 916865 T3842 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/shard-1-001/cores
   [junit4]   2> 916866 T3842 oasc.CoreContainer.<init> New CoreContainer 984802740
   [junit4]   2> 916866 T3842 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/shard-1-001/]
   [junit4]   2> 916867 T3842 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/shard-1-001/lib
   [junit4]   2> 916875 T3842 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/shard-1-001/lib).
   [junit4]   2> 916891 T3842 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 916892 T3842 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 916916 T3842 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 916916 T3842 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 916917 T3842 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 916917 T3842 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 916918 T3842 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 916918 T3842 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 916918 T3842 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 916919 T3842 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 916920 T3842 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 916920 T3842 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 916921 T3842 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 916921 T3842 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 916922 T3842 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 916922 T3842 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:58677/solr
   [junit4]   2> 916923 T3842 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 916923 T3842 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 916925 T3842 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 916927 T3895 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@57f4cb7c name:ZooKeeperConnection Watcher:127.0.0.1:58677 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 916928 T3842 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 916929 T3842 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 916930 T3842 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 916932 T3898 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5b892b77 name:ZooKeeperConnection Watcher:127.0.0.1:58677/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 916932 T3842 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 916938 T3842 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 917945 T3842 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:58686_
   [junit4]   2> 917946 T3842 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:58686_
   [junit4]   2> 917950 T3842 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 917953 T3842 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/shard-1-001/cores
   [junit4]   2> 917956 T3842 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {loadOnStartup=true, config=solrconfig.xml, collection=collection1, shard=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/shard-1-001/cores/collection1/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/shard-1-001/cores/collection1, schema=schema.xml, dataDir=data/, name=collection1, coreNodeName=, transient=false}
   [junit4]   2> 917956 T3842 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/shard-1-001/cores/collection1/
   [junit4]   2> 917957 T3842 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 917959 T3899 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 917959 T3899 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 917961 T3868 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 917961 T3899 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 917961 T3899 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 917962 T3869 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58686",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:58686_",
   [junit4]   2> 	  "state":"down"} current state version: 5
   [junit4]   2> 917962 T3899 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 917962 T3869 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58686",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:58686_",
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 917963 T3869 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 917963 T3869 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 917963 T3899 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 917963 T3899 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 917964 T3869 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 917965 T3898 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 917966 T3898 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 918964 T3899 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 918965 T3899 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 918965 T3899 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 918965 T3899 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 918966 T3899 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 918966 T3899 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/shard-1-001/cores/collection1/'
   [junit4]   2> 918982 T3899 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 918985 T3899 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 918996 T3899 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 919022 T3899 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 919063 T3899 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 919064 T3899 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 919072 T3899 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 919407 T3899 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 919425 T3899 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 919428 T3899 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 919440 T3899 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 919445 T3899 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 919450 T3899 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 919452 T3899 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 919452 T3899 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 919452 T3899 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 919453 T3899 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 919453 T3899 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 919454 T3899 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 919454 T3899 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 919454 T3899 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 919455 T3899 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@638c7ceb
   [junit4]   2> 919455 T3899 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/shard-1-001/cores/collection1/data
   [junit4]   2> 919456 T3899 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 919456 T3899 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 919456 T3899 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 919457 T3899 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1062300316, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 919458 T3899 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2213f549 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4c4c9d19),segFN=segments_1,generation=1}
   [junit4]   2> 919458 T3899 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 919464 T3899 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 919465 T3899 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 919465 T3899 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 919466 T3899 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 919466 T3899 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 919466 T3899 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 919467 T3899 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 919467 T3899 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 919468 T3899 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 919491 T3899 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 919494 T3899 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 919497 T3899 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 919500 T3899 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 919507 T3899 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/plugins,/schema,/config,/update/csv,/admin/system,/admin/segments,/admin/ping,/admin/luke,/admin/threads,/admin/logging,/admin/properties,standard,/update/json,/get,/replication,/update,/update/json/docs,/admin/file,/admin/mbeans
   [junit4]   2> 919509 T3899 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 919511 T3899 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 919511 T3899 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 919513 T3899 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 919513 T3899 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 919514 T3899 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=37, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 919514 T3899 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2213f549 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4c4c9d19),segFN=segments_1,generation=1}
   [junit4]   2> 919515 T3899 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 919515 T3899 oass.SolrIndexSearcher.<init> Opening Searcher@38436df2[collection1] main
   [junit4]   2> 919515 T3899 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 919516 T3899 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 919516 T3899 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 919517 T3899 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 919517 T3899 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 919517 T3899 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 919518 T3899 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 919518 T3899 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 919518 T3899 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 919519 T3899 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 919520 T3900 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@38436df2[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 919520 T3899 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 919522 T3903 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:58686 collection:collection1 shard:shard1
   [junit4]   2> 919523 T3842 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 919523 T3842 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 919537 T3903 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 919542 T3903 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 919544 T3868 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 919544 T3903 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 919545 T3903 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C531 name=collection1 org.apache.solr.core.SolrCore@40b380d3 url=http://127.0.0.1:58686/collection1 node=127.0.0.1:58686_ C531_STATE=coll:collection1 core:collection1 props:{core=collection1, base_url=http://127.0.0.1:58686, node_name=127.0.0.1:58686_, state=down}
   [junit4]   2> 919545 T3903 C531 P58686 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:58686/collection1/
   [junit4]   2> 919545 T3869 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 919546 T3903 C531 P58686 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 919546 T3903 C531 P58686 oasc.SyncStrategy.syncToMe http://127.0.0.1:58686/collection1/ has no replicas
   [junit4]   2> 919547 T3903 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:58686/collection1/ shard1
   [junit4]   2> 919547 T3903 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 919547 T3869 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 919548 T3898 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 919549 T3898 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 919552 T3868 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 919553 T3869 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58686",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 919554 T3869 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 919555 T3898 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 919556 T3898 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 919610 T3903 oasc.ZkController.register We are http://127.0.0.1:58686/collection1/ and leader is http://127.0.0.1:58686/collection1/
   [junit4]   2> 919611 T3903 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:58686
   [junit4]   2> 919611 T3903 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 919611 T3903 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 919612 T3903 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 919613 T3868 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 919614 T3869 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58686",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:58686_",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 919615 T3869 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58686",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:58686_",
   [junit4]   2> 	  "state":"active"}
   [junit4]   2> 919616 T3903 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 919617 T3869 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 3
   [junit4]   2> 919618 T3898 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 919618 T3898 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 4 
   [junit4]   2> 921344 T3842 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/shard-2-001/cores/collection1
   [junit4]   2> 921347 T3842 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/shard-2-001
   [junit4]   2> 921348 T3842 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 921352 T3842 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:58691
   [junit4]   2> 921352 T3842 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=58690, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/shard-2-001/cores, solrconfig=solrconfig.xml, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/tempDir-001/jetty2, hostContext=/}
   [junit4]   2> 921353 T3842 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 921353 T3842 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/shard-2-001/'
   [junit4]   2> 921414 T3842 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/shard-2-001/solr.xml
   [junit4]   2> 921494 T3842 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/shard-2-001/cores
   [junit4]   2> 921494 T3842 oasc.CoreContainer.<init> New CoreContainer 725629679
   [junit4]   2> 921495 T3842 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/shard-2-001/]
   [junit4]   2> 921495 T3842 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/shard-2-001/lib
   [junit4]   2> 921496 T3842 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest DBB40DFD5B28D458-001/shard-2-001/lib).
   [junit4]   2> 921511 T3842 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 

[...truncated too long message...]

ate.json
   [junit4]   2> 		5	/solr/aliases.json
   [junit4]   2> 		4	/solr/configs/conf1
   [junit4]   2> 		3	/solr/collections/collection1/state.json
   [junit4]   2> 		3	/solr/collections/repfacttest_c8n_1x3/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		5	/solr/live_nodes
   [junit4]   2> 		4	/solr/overseer/queue
   [junit4]   2> 		4	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1501673 T3131 oasc.SocketProxy.close WARN Closing 9 connections to: http://127.0.0.1:30074/i_pc
   [junit4]   2> 1501674 T3131 oasc.SocketProxy.close WARN Closing 2 connections to: http://127.0.0.1:30082/i_pc
   [junit4]   2> 1501674 T3131 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:30095/i_pc
   [junit4]   2> 1501675 T3131 oasc.SocketProxy.close WARN Closing 7 connections to: http://127.0.0.1:30124/i_pc
   [junit4]   2> 1501675 T3131 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 1501676 T3131 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:30074/i_pc
   [junit4]   2> 1501676 T3131 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:30082/i_pc
   [junit4]   2> 1501676 T3131 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:30095/i_pc
   [junit4]   2> 1501677 T3131 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:30124/i_pc
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=DBB40DFD5B28D458 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=de_AT -Dtests.timezone=Asia/Chungking -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR    119s J2 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([DBB40DFD5B28D458:53E03227F5D4B9A0]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 1501724 T3131 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest DBB40DFD5B28D458-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {multiDefault=FST50, _version_=Lucene50(blocksize=128), range_facet_l=Lucene50(blocksize=128), timestamp=Lucene50(blocksize=128), a_t=Lucene50(blocksize=128), text=PostingsFormat(name=MockRandom), intDefault=Lucene50(blocksize=128), id=PostingsFormat(name=LuceneFixedGap)}, docValues:{timestamp=DocValuesFormat(name=Memory)}, sim=DefaultSimilarity, locale=de_AT, timezone=Asia/Chungking
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=188279000,total=294125568
   [junit4]   2> NOTE: All tests run in this JVM: [JSONWriterTest, AnalyticsMergeStrategyTest, DateRangeFieldTest, DistributedQueryComponentOptimizationTest, TestGroupingSearch, TestReRankQParserPlugin, BasicZkTest, TestQueryUtils, TestLMJelinekMercerSimilarityFactory, TestWordDelimiterFilterFactory, TestSolrQueryParserDefaultOperatorResource, TestSimpleTrackingShardHandler, TermsComponentTest, TestSchemaResource, CurrencyFieldXmlFileTest, TestDistributedMissingSort, TestComponentsName, DistanceFunctionTest, CursorMarkTest, ShardSplitTest, TestSortingResponseWriter, SimpleFacetsTest, HdfsLockFactoryTest, AnalysisAfterCoreReloadTest, RemoteQueryErrorTest, TestFaceting, TestCoreDiscovery, TestFieldSortValues, TestBadConfig, TestHighlightDedupGrouping, ExternalFileFieldSortTest, SyncSliceTest, DistributedSuggestComponentTest, ResourceLoaderTest, TestQuerySenderListener, OverseerTest, TestFunctionQuery, TestPHPSerializedResponseWriter, RAMDirectoryFactoryTest, TestManagedSchemaFieldResource, HdfsBasicDistributedZk2Test, HardAutoCommitTest, DirectUpdateHandlerTest, CachingDirectoryFactoryTest, SuggesterTest, DocumentAnalysisRequestHandlerTest, TestSystemIdResolver, TestMiniSolrCloudCluster, TestSolr4Spatial2, TestDistributedGrouping, TestDynamicFieldCollectionResource, TestStressVersions, TestFoldingMultitermQuery, SpellPossibilityIteratorTest, BadCopyFieldTest, NumericFieldsTest, TestAddFieldRealTimeGet, TestConfigSets, HdfsWriteToMultipleCollectionsTest, LeaderFailoverAfterPartitionTest, ReplicationFactorTest]
   [junit4] Completed on J2 in 118.87s, 1 test, 1 error <<< FAILURES!

[...truncated 706 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 475 suites, 1892 tests, 3 errors, 62 ignored (22 assumptions)

Total time: 52 minutes 51 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 194757924 bytes
Compression is 0.0%
Took 35 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2821 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2821/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:41194/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:41194/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([B3FAB110106DBD5:836B94CBAFFAB62D]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([B3FAB110106DBD5:836B94CBAFFAB62D]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([B3FAB110106DBD5:836B94CBAFFAB62D]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9762 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/init-core-data-001
   [junit4]   2> 702932 T894 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 702940 T894 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 702940 T895 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 703040 T894 oasc.ZkTestServer.run start zk server on port:37104
   [junit4]   2> 703041 T894 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 703042 T894 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 703046 T902 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@78afa4c8 name:ZooKeeperConnection Watcher:127.0.0.1:37104 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 703047 T894 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 703048 T894 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 703048 T894 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 703051 T894 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 703052 T894 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 703054 T905 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2479d23e name:ZooKeeperConnection Watcher:127.0.0.1:37104/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 703054 T894 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 703055 T894 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 703055 T894 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 703058 T894 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 703060 T894 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 703062 T894 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 703063 T894 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 703064 T894 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 703066 T894 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 703067 T894 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 703169 T894 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 703169 T894 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 703171 T894 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 703172 T894 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 703173 T894 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 703174 T894 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 703176 T894 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 703176 T894 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 703178 T894 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 703179 T894 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 703181 T894 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 703181 T894 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 703183 T894 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 703184 T894 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 703186 T894 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 703186 T894 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 703188 T894 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 703188 T894 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 704321 T894 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/control-001/cores/collection1
   [junit4]   2> 704323 T894 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 704327 T894 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:60660
   [junit4]   2> 704327 T894 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/tempDir-001/control/data, hostPort=20239, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/control-001/cores, hostContext=/}
   [junit4]   2> 704328 T894 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 704328 T894 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/control-001/'
   [junit4]   2> 704357 T894 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/control-001/solr.xml
   [junit4]   2> 704393 T894 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/control-001/cores
   [junit4]   2> 704393 T894 oasc.CoreContainer.<init> New CoreContainer 2012921520
   [junit4]   2> 704393 T894 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/control-001/]
   [junit4]   2> 704393 T894 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/control-001/lib
   [junit4]   2> 704394 T894 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/control-001/lib).
   [junit4]   2> 704403 T894 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 704403 T894 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 704403 T894 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 704403 T894 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 704404 T894 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 704404 T894 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 704404 T894 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 704404 T894 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 704404 T894 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 704405 T894 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 704405 T894 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 704405 T894 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 704406 T894 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 704406 T894 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 704406 T894 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 704406 T894 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:37104/solr
   [junit4]   2> 704406 T894 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 704407 T894 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 704407 T894 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 704409 T917 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@668c32ff name:ZooKeeperConnection Watcher:127.0.0.1:37104 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 704409 T894 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 704410 T894 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 704412 T894 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 704413 T920 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1c103a14 name:ZooKeeperConnection Watcher:127.0.0.1:37104/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 704413 T894 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 704414 T894 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 704416 T894 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 704417 T894 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 704419 T894 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 704420 T894 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 704422 T894 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 704423 T894 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 704424 T894 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 704425 T894 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:20239_
   [junit4]   2> 704425 T894 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:20239_
   [junit4]   2> 704426 T894 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 704427 T894 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 704428 T894 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 704429 T894 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:20239_
   [junit4]   2> 704430 T894 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 704431 T894 oasc.Overseer.start Overseer (id=93528775407370243-127.0.0.1:20239_-n_0000000000) starting
   [junit4]   2> 704432 T894 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 704437 T894 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 704437 T922 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 704437 T894 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 704437 T921 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 704440 T894 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/control-001/cores
   [junit4]   2> 704442 T894 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {shard=, dataDir=data/, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/control-001/cores/collection1/, coreNodeName=, collection=control_collection, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, name=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/control-001/cores/collection1}
   [junit4]   2> 704442 T894 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/control-001/cores/collection1/
   [junit4]   2> 704442 T894 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 704443 T924 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 704443 T924 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 704444 T920 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 704444 T924 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 704444 T921 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:20239_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20239",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 0
   [junit4]   2> 704445 T921 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:20239_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20239",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"control_collection"}
   [junit4]   2> 704445 T921 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 704445 T921 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 704446 T920 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 705444 T924 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 705445 T924 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 705445 T924 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 705446 T924 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 705446 T924 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 705446 T924 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/control-001/cores/collection1/'
   [junit4]   2> 705465 T924 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 705468 T924 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 705481 T924 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 705507 T924 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 705564 T924 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 705665 T924 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 705678 T924 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 706127 T924 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 706148 T924 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 706151 T924 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 706164 T924 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 706169 T924 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 706174 T924 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 706175 T924 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 706175 T924 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 706175 T924 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 706176 T924 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 706176 T924 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 706177 T924 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 706177 T924 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 706177 T924 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/control-001/cores/collection1/data/
   [junit4]   2> 706178 T924 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4e9db371
   [junit4]   2> 706179 T924 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/control-001/cores/collection1/data
   [junit4]   2> 706179 T924 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/control-001/cores/collection1/data/index/
   [junit4]   2> 706179 T924 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 706180 T924 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/control-001/cores/collection1/data/index
   [junit4]   2> 706181 T924 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=50, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 706181 T924 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@14455abb lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5547b562),segFN=segments_1,generation=1}
   [junit4]   2> 706182 T924 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 706187 T924 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 706188 T924 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 706188 T924 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 706188 T924 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 706189 T924 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 706189 T924 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 706189 T924 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 706190 T924 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 706190 T924 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 706211 T924 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 706214 T924 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 706216 T924 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 706218 T924 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 706224 T924 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update/csv,/update,/config,/admin/system,/update/json,/get,/admin/properties,/schema,standard,/admin/mbeans,/replication,/admin/file,/admin/logging,/admin/threads,/admin/plugins,/update/json/docs,/admin/luke,/admin/ping,/admin/segments
   [junit4]   2> 706226 T924 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 706227 T924 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 706228 T924 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 706229 T924 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 706229 T924 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 706230 T924 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=50, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 706231 T924 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@14455abb lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5547b562),segFN=segments_1,generation=1}
   [junit4]   2> 706231 T924 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 706231 T924 oass.SolrIndexSearcher.<init> Opening Searcher@1ab5f695[collection1] main
   [junit4]   2> 706231 T924 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 706232 T924 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 706233 T924 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 706233 T924 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 706233 T924 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 706233 T924 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 706234 T924 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 706234 T924 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 706235 T924 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 706235 T924 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 706235 T925 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1ab5f695[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 706236 T924 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 706237 T928 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:20239 collection:control_collection shard:shard1
   [junit4]   2> 706237 T894 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 706237 T928 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 706237 T894 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 706239 T894 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 706239 T894 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 706241 T932 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@434aec15 name:ZooKeeperConnection Watcher:127.0.0.1:37104/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 706241 T894 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 706242 T894 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 706242 T928 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 706242 T894 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 706244 T920 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 706244 T928 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 706244 T928 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C191 name=collection1 org.apache.solr.core.SolrCore@2059dc48 url=http://127.0.0.1:20239/collection1 node=127.0.0.1:20239_ C191_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:20239_, base_url=http://127.0.0.1:20239, core=collection1, state=down}
   [junit4]   2> 706244 T928 C191 P20239 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:20239/collection1/
   [junit4]   2> 706244 T921 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 706244 T928 C191 P20239 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 706245 T894 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 706245 T928 C191 P20239 oasc.SyncStrategy.syncToMe http://127.0.0.1:20239/collection1/ has no replicas
   [junit4]   2> 706245 T928 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:20239/collection1/ shard1
   [junit4]   2> 706245 T928 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 706246 T932 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 706246 T920 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 706248 T920 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 706249 T921 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20239",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 706352 T932 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 706352 T920 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 706401 T928 oasc.ZkController.register We are http://127.0.0.1:20239/collection1/ and leader is http://127.0.0.1:20239/collection1/
   [junit4]   2> 706402 T928 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:20239
   [junit4]   2> 706402 T928 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 706402 T928 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 706402 T928 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 706403 T920 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 706404 T921 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:20239_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20239",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 3
   [junit4]   2> 706405 T921 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:20239_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20239",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"control_collection"}
   [junit4]   2> 706507 T932 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 706507 T920 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 707397 T894 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/shard-1-001/cores/collection1
   [junit4]   2> 707401 T894 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/shard-1-001
   [junit4]   2> 707401 T894 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 707403 T894 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:27156
   [junit4]   2> 707403 T894 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/shard-1-001/cores, hostPort=45682, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/}
   [junit4]   2> 707404 T894 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 707404 T894 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/shard-1-001/'
   [junit4]   2> 707442 T894 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/shard-1-001/solr.xml
   [junit4]   2> 707495 T894 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/shard-1-001/cores
   [junit4]   2> 707495 T894 oasc.CoreContainer.<init> New CoreContainer 772759674
   [junit4]   2> 707495 T894 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/shard-1-001/]
   [junit4]   2> 707496 T894 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/shard-1-001/lib
   [junit4]   2> 707496 T894 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/shard-1-001/lib).
   [junit4]   2> 707506 T894 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 707507 T894 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 707507 T894 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 707507 T894 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 707507 T894 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 707507 T894 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 707508 T894 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 707508 T894 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 707508 T894 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 707508 T894 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 707508 T894 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 707509 T894 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 707509 T894 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 707509 T894 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 707510 T894 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 707510 T894 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:37104/solr
   [junit4]   2> 707510 T894 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 707510 T894 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 707511 T894 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 707512 T944 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1b1c8149 name:ZooKeeperConnection Watcher:127.0.0.1:37104 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 707513 T894 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 707514 T894 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 707515 T894 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 707516 T947 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@557fed47 name:ZooKeeperConnection Watcher:127.0.0.1:37104/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 707516 T894 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 707519 T894 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 708523 T894 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45682_
   [junit4]   2> 708524 T894 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:45682_
   [junit4]   2> 708526 T894 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 708529 T894 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/shard-1-001/cores
   [junit4]   2> 708532 T894 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {dataDir=data/, transient=false, schema=schema.xml, collection=collection1, shard=, name=collection1, config=solrconfig.xml, loadOnStartup=true, coreNodeName=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/shard-1-001/cores/collection1/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/shard-1-001/cores/collection1}
   [junit4]   2> 708532 T894 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/shard-1-001/cores/collection1/
   [junit4]   2> 708532 T894 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 708533 T948 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 708533 T948 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 708534 T948 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 708534 T920 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 708535 T921 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:45682_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:45682",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"collection1"} current state version: 4
   [junit4]   2> 708536 T921 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:45682_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:45682",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"collection1"}
   [junit4]   2> 708536 T921 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 708536 T921 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 708537 T932 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 708537 T920 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 708537 T947 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 709535 T948 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 709536 T948 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 709536 T948 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 709537 T948 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 709537 T948 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 709537 T948 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/shard-1-001/cores/collection1/'
   [junit4]   2> 709555 T948 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 709557 T948 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 709568 T948 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 709592 T948 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 709643 T948 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 709744 T948 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 709752 T948 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 710063 T948 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 710077 T948 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 710079 T948 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 710088 T948 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 710092 T948 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 710095 T948 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 710095 T948 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 710095 T948 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 710096 T948 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 710096 T948 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 710096 T948 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 710096 T948 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 710097 T948 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 710097 T948 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 710097 T948 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4e9db371
   [junit4]   2> 710098 T948 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/shard-1-001/cores/collection1/data
   [junit4]   2> 710098 T948 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 710098 T948 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 710098 T948 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 710098 T948 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=50, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 710099 T948 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@18082755 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3f4e49c6),segFN=segments_1,generation=1}
   [junit4]   2> 710099 T948 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 710104 T948 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 710104 T948 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 710104 T948 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 710105 T948 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 710105 T948 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 710105 T948 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 710105 T948 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 710105 T948 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 710106 T948 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 710124 T948 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 710125 T948 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 710127 T948 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 710129 T948 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 710135 T948 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update/csv,/update,/config,/admin/system,/update/json,/get,/admin/properties,/schema,standard,/admin/mbeans,/replication,/admin/file,/admin/logging,/admin/threads,/admin/plugins,/update/json/docs,/admin/luke,/admin/ping,/admin/segments
   [junit4]   2> 710136 T948 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 710138 T948 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 710138 T948 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 710139 T948 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 710139 T948 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 710140 T948 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=50, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 710140 T948 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@18082755 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3f4e49c6),segFN=segments_1,generation=1}
   [junit4]   2> 710141 T948 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 710141 T948 oass.SolrIndexSearcher.<init> Opening Searcher@faf053e[collection1] main
   [junit4]   2> 710141 T948 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 710142 T948 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 710142 T948 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 710142 T948 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 710142 T948 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 710142 T948 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 710143 T948 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 710143 T948 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 710143 T948 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 710143 T948 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 710144 T949 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@faf053e[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 710144 T948 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 710145 T952 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:45682 collection:collection1 shard:shard1
   [junit4]   2> 710145 T894 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 710145 T894 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 710146 T952 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 710149 T952 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 710150 T920 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 710150 T952 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 710150 T952 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C192 name=collection1 org.apache.solr.core.SolrCore@545d9543 url=http://127.0.0.1:45682/collection1 node=127.0.0.1:45682_ C192_STATE=coll:collection1 core:collection1 props:{node_name=127.0.0.1:45682_, base_url=http://127.0.0.1:45682, core=collection1, state=down}
   [junit4]   2> 710150 T952 C192 P45682 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:45682/collection1/
   [junit4]   2> 710150 T921 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 710150 T952 C192 P45682 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 710150 T952 C192 P45682 oasc.SyncStrategy.syncToMe http://127.0.0.1:45682/collection1/ has no replicas
   [junit4]   2> 710151 T952 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:45682/collection1/ shard1
   [junit4]   2> 710151 T952 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 710151 T932 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 710151 T920 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 710151 T947 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 710154 T920 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 710154 T921 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:45682",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 710257 T947 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 710257 T932 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 710257 T920 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 710306 T952 oasc.ZkController.register We are http://127.0.0.1:45682/collection1/ and leader is http://127.0.0.1:45682/collection1/
   [junit4]   2> 710307 T952 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:45682
   [junit4]   2> 710307 T952 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 710307 T952 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 710307 T952 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 710308 T920 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 710308 T921 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:45682_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:45682",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 7
   [junit4]   2> 710309 T921 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:45682_",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:45682",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"collection1"}
   [junit4]   2> 710411 T920 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 710411 T947 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 710411 T932 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 711099 T894 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/shard-2-001/cores/collection1
   [junit4]   2> 711101 T894 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/shard-2-001
   [junit4]   2> 711101 T894 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 711104 T894 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:38651
   [junit4]   2> 711104 T894 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solrconfig=solrconfig.xml, hostPort=41194, hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/shard-2-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/tempDir-001/jetty2}
   [junit4]   2> 711104 T894 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 711104 T894 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/shard-2-001/'
   [junit4]   2> 711134 T894 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/shard-2-001/solr.xml
   [junit4]   2> 711170 T894 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/shard-2-001/cores
   [junit4]   2> 711170 T894 oasc.CoreContainer.<init> New CoreContainer 2050880580
   [junit4]   2> 711170 T894 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/shard-2-001/]
   [junit4]   2> 711171 T894 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/shard-2-001/lib
   [junit4]   2> 711171 T894 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest B3FAB110106DBD5-001/shard-2-001/lib).
   [junit4]   2> 711180 T894 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 711180 T894 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 711181 T894 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 711181 T894 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 711181 T894 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 711181 T894 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 711181 T894 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 711182 T894 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 711182 T894 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 711182 T894 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 711182 T894 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 711183 T894 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 711183 T894 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 711183 T894 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 711183 T894 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 711184 T894 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:37104/solr
   [junit4]   2> 711184 T894 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 711184 T894 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 711185 T894 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 711186 T965 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@63c882b5 name:ZooKeeperConnection Watcher:127.0.0.1:37104 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 711187 T894 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 711187 T894 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 711189 T894 oascc.Connection

[...truncated too long message...]

cess WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 1483515 T3584 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> 1483616 T3584 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:26911 26911
   [junit4]   2> 1483723 T3585 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:26911 26911
   [junit4]   2> 1483724 T3585 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 1483724 T3584 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:26953/
   [junit4]   2> 1483724 T3584 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:26936/
   [junit4]   2> 1483725 T3584 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:26918/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=B3FAB110106DBD5 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ar_DZ -Dtests.timezone=Africa/Ouagadougou -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   67.1s J2 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([B3FAB110106DBD5:836B94CBAFFAB62D]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 1483734 T3584 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest B3FAB110106DBD5-001
   [junit4]   2> 67088 T3583 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {multiDefault=PostingsFormat(name=LuceneFixedGap), intDefault=PostingsFormat(name=MockRandom), timestamp=PostingsFormat(name=MockRandom), a_t=PostingsFormat(name=MockRandom), range_facet_l=PostingsFormat(name=MockRandom), id=PostingsFormat(name=Memory doPackFST= false), _version_=PostingsFormat(name=MockRandom), text=FST50}, docValues:{timestamp=DocValuesFormat(name=Direct)}, sim=RandomSimilarityProvider(queryNorm=false,coord=yes): {}, locale=ar_DZ, timezone=Africa/Ouagadougou
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=190023448,total=415236096
   [junit4]   2> NOTE: All tests run in this JVM: [TestIBSimilarityFactory, TestAtomicUpdateErrorCases, RequiredFieldsTest, SimpleFacetsTest, TestSolrJ, ExternalFileFieldSortTest, AddBlockUpdateTest, TestClassNameShortening, TestSystemIdResolver, TestDynamicFieldResource, TestSolrDynamicMBean, SolrXmlInZkTest, DirectUpdateHandlerTest, TestQuerySenderNoQuery, TestObjectReleaseTracker, ResourceLoaderTest, TestBulkSchemaAPI, TestJsonRequest, DistributedQueryComponentOptimizationTest, LeaderElectionIntegrationTest, TestFreeTextSuggestions, SpellPossibilityIteratorTest, BlockDirectoryTest, TestZkChroot, PrimitiveFieldTypeTest, ShardSplitTest, TestBlobHandler, CursorMarkTest, XsltUpdateRequestHandlerTest, TestCSVResponseWriter, TestRawResponseWriter, DistributedQueryComponentCustomSortTest, TestSolrConfigHandlerCloud, DistributedFacetPivotWhiteBoxTest, TestChildDocTransformer, SearchHandlerTest, TestCloudSchemaless, TestFiltering, TestFunctionQuery, TestLuceneMatchVersion, StandardRequestHandlerTest, RequestLoggingTest, TestReload, NoCacheHeaderTest, TestCollationField, TestIndexSearcher, TestSchemaResource, TestBM25SimilarityFactory, AddSchemaFieldsUpdateProcessorFactoryTest, TestRestManager, OverriddenZkACLAndCredentialsProvidersTest, PeerSyncTest, TestQueryUtils, TestXIncludeConfig, TestTrackingShardHandlerFactory, SolrIndexSplitterTest, SyncSliceTest, TestDistributedSearch, BasicDistributedZkTest, ReplicationFactorTest, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J2 in 67.87s, 1 test, 1 error <<< FAILURES!

[...truncated 692 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 475 suites, 1892 tests, 3 errors, 62 ignored (22 assumptions)

Total time: 52 minutes 35 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 221848150 bytes
Compression is 0.0%
Took 41 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-5.x-Java7 - Build # 2820 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-5.x-Java7/2820/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:21601/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:21601/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([20B7E01448DCCF36:A8E3DFCEE620A2CE]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:29162/cp_fk/y/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:29162/cp_fk/y/collection1
	at __randomizedtesting.SeedInfo.seed([20B7E01448DCCF36:A8E3DFCEE620A2CE]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:29162/cp_fk/y/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28973/repfacttest_c8n_1x3_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28973/repfacttest_c8n_1x3_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([20B7E01448DCCF36:A8E3DFCEE620A2CE]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28973/repfacttest_c8n_1x3_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 53 more




Build Log:
[...truncated 9241 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/init-core-data-001
   [junit4]   2> 43994 T85 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 44072 T85 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 44088 T86 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 44390 T85 oasc.ZkTestServer.run start zk server on port:58868
   [junit4]   2> 44577 T85 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 44812 T85 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 45009 T93 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7a2c3b80 name:ZooKeeperConnection Watcher:127.0.0.1:58868 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 45010 T85 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 45011 T85 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 45024 T85 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 45145 T85 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 45146 T85 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 45148 T96 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@297978c5 name:ZooKeeperConnection Watcher:127.0.0.1:58868/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 45148 T85 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 45149 T85 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 45169 T85 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 45173 T85 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 45177 T85 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 45179 T85 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 45183 T85 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 45183 T85 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 45188 T85 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 45189 T85 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 45193 T85 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 45193 T85 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 45197 T85 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 45198 T85 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 45201 T85 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 45202 T85 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 45206 T85 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 45206 T85 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 45210 T85 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 45211 T85 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 45214 T85 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 45215 T85 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 45218 T85 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 45218 T85 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 45229 T85 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 45230 T85 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 45234 T85 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 45235 T85 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 47024 T85 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/control-001/cores/collection1
   [junit4]   2> 47031 T85 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 47035 T85 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:22514
   [junit4]   2> 47036 T85 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/control-001/cores, hostPort=28529, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/tempDir-001/control/data}
   [junit4]   2> 47037 T85 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 47038 T85 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/control-001/'
   [junit4]   2> 47101 T85 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/control-001/solr.xml
   [junit4]   2> 47186 T85 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/control-001/cores
   [junit4]   2> 47187 T85 oasc.CoreContainer.<init> New CoreContainer 738349763
   [junit4]   2> 47188 T85 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/control-001/]
   [junit4]   2> 47188 T85 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/control-001/lib
   [junit4]   2> 47190 T85 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/control-001/lib).
   [junit4]   2> 47206 T85 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 47207 T85 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 47207 T85 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 47208 T85 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 47208 T85 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 47209 T85 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 47210 T85 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 47210 T85 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 47211 T85 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 47211 T85 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 47212 T85 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 47213 T85 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 47214 T85 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 47214 T85 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 47215 T85 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 47216 T85 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:58868/solr
   [junit4]   2> 47294 T85 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 47295 T85 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 47296 T85 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 47302 T108 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@60d96f5b name:ZooKeeperConnection Watcher:127.0.0.1:58868 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 47303 T85 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 47311 T85 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 47358 T85 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 47361 T111 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6d0433ee name:ZooKeeperConnection Watcher:127.0.0.1:58868/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 47362 T85 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 47388 T85 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 47411 T85 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 47426 T85 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 47431 T85 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 47437 T85 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 47474 T85 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 47478 T85 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 47482 T85 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 47485 T85 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:28529_
   [junit4]   2> 47495 T85 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:28529_
   [junit4]   2> 47500 T85 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 47505 T85 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 47509 T85 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 47537 T85 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:28529_
   [junit4]   2> 47538 T85 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 47542 T85 oasc.Overseer.start Overseer (id=93527275490639875-127.0.0.1:28529_-n_0000000000) starting
   [junit4]   2> 47569 T85 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 47606 T85 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 47964 T113 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 47965 T85 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 47991 T112 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 47997 T85 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/control-001/cores
   [junit4]   2> 48017 T85 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=control_collection, config=solrconfig.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/control-001/cores/collection1/, shard=, loadOnStartup=true, transient=false, coreNodeName=, dataDir=data/, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/control-001/cores/collection1, name=collection1}
   [junit4]   2> 48018 T85 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/control-001/cores/collection1/
   [junit4]   2> 48018 T85 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 48019 T115 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 48020 T115 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 48040 T111 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 48041 T115 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 48055 T112 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28529",
   [junit4]   2> 	  "node_name":"127.0.0.1:28529_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null} current state version: 0
   [junit4]   2> 48207 T112 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28529",
   [junit4]   2> 	  "node_name":"127.0.0.1:28529_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 48221 T112 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 48313 T112 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 48340 T111 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 49042 T115 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 49042 T115 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 49043 T115 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 49043 T115 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 49045 T115 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 49045 T115 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/control-001/cores/collection1/'
   [junit4]   2> 49096 T115 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 49121 T115 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 49163 T115 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 49193 T115 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 49246 T115 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 49248 T115 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 49287 T115 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 50454 T115 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 50505 T115 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 50508 T115 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 50578 T115 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 50584 T115 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 50589 T115 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 50596 T115 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 50596 T115 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 50597 T115 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 50598 T115 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 50599 T115 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 50610 T115 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 50611 T115 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 50612 T115 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/control-001/cores/collection1/data/
   [junit4]   2> 50612 T115 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@881229a
   [junit4]   2> 50624 T115 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/control-001/cores/collection1/data
   [junit4]   2> 50625 T115 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/control-001/cores/collection1/data/index/
   [junit4]   2> 50625 T115 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 50625 T115 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/control-001/cores/collection1/data/index
   [junit4]   2> 50626 T115 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=45, maxMergeAtOnceExplicit=45, maxMergedSegmentMB=37.3115234375, floorSegmentMB=1.9853515625, forceMergeDeletesPctAllowed=0.5863137249875605, segmentsPerTier=38.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 50627 T115 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5aec6584 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55fcebbe),segFN=segments_1,generation=1}
   [junit4]   2> 50627 T115 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 50637 T115 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 50750 T115 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 50751 T115 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 50751 T115 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 50752 T115 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 50752 T115 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 50753 T115 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 50753 T115 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 50753 T115 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 50770 T115 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 50781 T115 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 50783 T115 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 50786 T115 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 50830 T115 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/luke,/replication,/update/json/docs,/update/csv,/schema,/admin/file,/update,/admin/plugins,/admin/mbeans,/admin/threads,standard,/admin/system,/admin/logging,/update/json,/config,/get,/admin/segments,/admin/properties,/admin/ping
   [junit4]   2> 50836 T115 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 50838 T115 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 50838 T115 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 50839 T115 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 50840 T115 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 50841 T115 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=17, maxMergeAtOnceExplicit=41, maxMergedSegmentMB=54.0341796875, floorSegmentMB=2.005859375, forceMergeDeletesPctAllowed=24.45973779119211, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4896825076899012
   [junit4]   2> 50842 T115 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5aec6584 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55fcebbe),segFN=segments_1,generation=1}
   [junit4]   2> 50842 T115 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 50843 T115 oass.SolrIndexSearcher.<init> Opening Searcher@61e04742[collection1] main
   [junit4]   2> 50843 T115 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 50845 T115 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 50845 T115 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 50855 T115 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 50855 T115 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 50855 T115 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 50857 T115 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 50857 T115 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 50857 T115 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 50858 T115 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 50861 T116 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@61e04742[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 50870 T115 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 50871 T119 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:28529 collection:control_collection shard:shard1
   [junit4]   2> 50872 T85 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 50872 T85 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 50894 T119 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 50904 T119 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 50907 T111 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 50908 T85 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 50908 T119 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 50908 T119 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 50908 T85 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> ASYNC  NEW_CORE C4 name=collection1 org.apache.solr.core.SolrCore@779ffee6 url=http://127.0.0.1:28529/collection1 node=127.0.0.1:28529_ C4_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:28529, node_name=127.0.0.1:28529_, core=collection1}
   [junit4]   2> 50909 T119 C4 P28529 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:28529/collection1/
   [junit4]   2> 50909 T112 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 50910 T119 C4 P28529 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 50911 T119 C4 P28529 oasc.SyncStrategy.syncToMe http://127.0.0.1:28529/collection1/ has no replicas
   [junit4]   2> 50911 T123 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@35ae2e31 name:ZooKeeperConnection Watcher:127.0.0.1:58868/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 50911 T119 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:28529/collection1/ shard1
   [junit4]   2> 50912 T85 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 50913 T85 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 50914 T85 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 50923 T85 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 50923 T85 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 50924 T85 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 50925 T85 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 50925 T119 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 50926 T123 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 50926 T111 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 50927 T126 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5fe10606 name:ZooKeeperConnection Watcher:127.0.0.1:58868/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 50928 T85 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 50929 T85 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 50938 T112 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 50939 T112 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 50939 T111 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 50947 T112 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28529",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 50949 T112 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 50952 T123 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 50952 T111 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 50961 T127 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 50964 T123 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 50964 T111 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 50994 T119 oasc.ZkController.register We are http://127.0.0.1:28529/collection1/ and leader is http://127.0.0.1:28529/collection1/
   [junit4]   2> 50994 T119 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:28529
   [junit4]   2> 50994 T119 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 50995 T119 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 50995 T119 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 50997 T111 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 50999 T112 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28529",
   [junit4]   2> 	  "node_name":"127.0.0.1:28529_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null} current state version: 4
   [junit4]   2> 51000 T112 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28529",
   [junit4]   2> 	  "node_name":"127.0.0.1:28529_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 51106 T111 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 51106 T123 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 52407 T85 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/shard-1-001/cores/collection1
   [junit4]   2> 52411 T85 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/shard-1-001
   [junit4]   2> 52412 T85 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 52414 T85 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:58467
   [junit4]   2> 52414 T85 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/shard-1-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=21601}
   [junit4]   2> 52415 T85 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 52415 T85 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/shard-1-001/'
   [junit4]   2> 52454 T85 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/shard-1-001/solr.xml
   [junit4]   2> 52506 T85 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/shard-1-001/cores
   [junit4]   2> 52507 T85 oasc.CoreContainer.<init> New CoreContainer 1559610902
   [junit4]   2> 52507 T85 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/shard-1-001/]
   [junit4]   2> 52507 T85 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/shard-1-001/lib
   [junit4]   2> 52508 T85 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/shard-1-001/lib).
   [junit4]   2> 52519 T85 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 52519 T85 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 52520 T85 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 52520 T85 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 52520 T85 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 52521 T85 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 52521 T85 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 52521 T85 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 52522 T85 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 52522 T85 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 52522 T85 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 52523 T85 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 52523 T85 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 52524 T85 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 52524 T85 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 52525 T85 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:58868/solr
   [junit4]   2> 52525 T85 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 52525 T85 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 52526 T85 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 52528 T139 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3beebf75 name:ZooKeeperConnection Watcher:127.0.0.1:58868 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 52529 T85 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 52530 T85 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 52532 T85 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 52534 T142 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@234172c8 name:ZooKeeperConnection Watcher:127.0.0.1:58868/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 52534 T85 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 52542 T85 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 53567 T85 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:21601_
   [junit4]   2> 53569 T85 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:21601_
   [junit4]   2> 53575 T85 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 53590 T85 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/shard-1-001/cores
   [junit4]   2> 53594 T85 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {coreNodeName=, collection=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/shard-1-001/cores/collection1/, dataDir=data/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/shard-1-001/cores/collection1, loadOnStartup=true, config=solrconfig.xml, shard=, transient=false, name=collection1, schema=schema.xml}
   [junit4]   2> 53595 T85 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/shard-1-001/cores/collection1/
   [junit4]   2> 53595 T85 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 53600 T143 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 53601 T143 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 53604 T111 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 53606 T143 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 53606 T112 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:21601",
   [junit4]   2> 	  "node_name":"127.0.0.1:21601_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null} current state version: 5
   [junit4]   2> 53606 T143 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 53608 T112 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:21601",
   [junit4]   2> 	  "node_name":"127.0.0.1:21601_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 53608 T143 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 53608 T112 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 53609 T112 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 53612 T112 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 53622 T143 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 53623 T143 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 53623 T143 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 53624 T143 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 53625 T143 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 53625 T143 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 53627 T143 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 53627 T143 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/shard-1-001/cores/collection1/'
   [junit4]   2> 53648 T143 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 53653 T143 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 53677 T143 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 53712 T143 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 53777 T143 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 53779 T143 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 53819 T143 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 54272 T143 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 54301 T143 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 54305 T143 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 54340 T143 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 54347 T143 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 54355 T143 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 54357 T143 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 54357 T143 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 54358 T143 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 54360 T143 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 54360 T143 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 54361 T143 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 54361 T143 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 54361 T143 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 54362 T143 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@881229a
   [junit4]   2> 54363 T143 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/shard-1-001/cores/collection1/data
   [junit4]   2> 54363 T143 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 54364 T143 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 54365 T143 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 54365 T143 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=45, maxMergeAtOnceExplicit=45, maxMergedSegmentMB=37.3115234375, floorSegmentMB=1.9853515625, forceMergeDeletesPctAllowed=0.5863137249875605, segmentsPerTier=38.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 54366 T143 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7ef769c1 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@25403f06),segFN=segments_1,generation=1}
   [junit4]   2> 54366 T143 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 54378 T143 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 54379 T143 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 54379 T143 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 54379 T143 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 54380 T143 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 54380 T143 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 54381 T143 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 54381 T143 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 54381 T143 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 54401 T143 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 54405 T143 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 54409 T143 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 54412 T143 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 54447 T143 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/luke,/replication,/update/json/docs,/update/csv,/schema,/admin/file,/update,/admin/plugins,/admin/mbeans,/admin/threads,standard,/admin/system,/admin/logging,/update/json,/config,/get,/admin/segments,/admin/properties,/admin/ping
   [junit4]   2> 54450 T143 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 54452 T143 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 54452 T143 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 54454 T143 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 54454 T143 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 54457 T143 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=17, maxMergeAtOnceExplicit=41, maxMergedSegmentMB=54.0341796875, floorSegmentMB=2.005859375, forceMergeDeletesPctAllowed=24.45973779119211, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4896825076899012
   [junit4]   2> 54458 T143 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7ef769c1 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@25403f06),segFN=segments_1,generation=1}
   [junit4]   2> 54458 T143 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 54459 T143 oass.SolrIndexSearcher.<init> Opening Searcher@3c3ad0de[collection1] main
   [junit4]   2> 54459 T143 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 54462 T143 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 54462 T143 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 54463 T143 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 54463 T143 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 54464 T143 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 54465 T143 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 54465 T143 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 54465 T143 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 54466 T143 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 54469 T144 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3c3ad0de[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 54469 T143 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 54470 T147 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:21601 collection:collection1 shard:shard1
   [junit4]   2> 54471 T85 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 54472 T85 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 54472 T147 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 54483 T147 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 54487 T111 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 54488 T147 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 54489 T147 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C5 name=collection1 org.apache.solr.core.SolrCore@29c8676f url=http://127.0.0.1:21601/collection1 node=127.0.0.1:21601_ C5_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:21601, node_name=127.0.0.1:21601_, core=collection1}
   [junit4]   2> 54489 T147 C5 P21601 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:21601/collection1/
   [junit4]   2> 54490 T147 C5 P21601 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 54490 T147 C5 P21601 oasc.SyncStrategy.syncToMe http://127.0.0.1:21601/collection1/ has no replicas
   [junit4]   2> 54491 T112 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 54491 T147 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:21601/collection1/ shard1
   [junit4]   2> 54492 T147 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 54494 T112 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 54502 T112 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:21601",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 54504 T112 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 54509 T111 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 54519 T142 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 54520 T142 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 54552 T147 oasc.ZkController.register We are http://127.0.0.1:21601/collection1/ and leader is http://127.0.0.1:21601/collection1/
   [junit4]   2> 54553 T147 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:21601
   [junit4]   2> 54553 T147 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 54554 T147 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 54554 T147 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 54556 T111 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 54558 T112 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:21601",
   [junit4]   2> 	  "node_name":"127.0.0.1:21601_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null} current state version: 5
   [junit4]   2> 54560 T112 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:21601",
   [junit4]   2> 	  "node_name":"127.0.0.1:21601_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 54562 T147 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 54563 T112 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 3
   [junit4]   2> 54565 T142 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 54566 T142 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 4 
   [junit4]   2> 56537 T85 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/shard-2-001/cores/collection1
   [junit4]   2> 56541 T85 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/shard-2-001
   [junit4]   2> 56542 T85 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 56544 T85 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:47139
   [junit4]   2> 56544 T85 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/shard-2-001/cores, hostContext=/, solrconfig=solrconfig.xml, hostPort=26903, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/tempDir-001/jetty2}
   [junit4]   2> 56545 T85 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 56546 T85 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/shard-2-001/'
   [junit4]   2> 56596 T85 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/shard-2-001/solr.xml
   [junit4]   2> 56665 T85 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/shard-2-001/cores
   [junit4]   2> 56666 T85 oasc.CoreContainer.<init> New CoreContainer 1466877398
   [junit4]   2> 56666 T85 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/shard-2-001/]
   [junit4]   2> 56667 T85 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/shard-2-001/lib
   [junit4]   2> 56668 T85 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 20B7E01448DCCF36-001/shard-2-001/lib).
   [junit4]   2> 56684 T85 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 56684 T85 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 56685 T85 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 56685 T85 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 56686 T85 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 56686 T85 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 56687 T85 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 56688 T85 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 56688 T85 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 56689 T85 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 56689 T85 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 56690 T85 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with 

[...truncated too long message...]

ncurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		5	/solr/live_nodes
   [junit4]   2> 		4	/solr/overseer/queue
   [junit4]   2> 		4	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1519378 T3949 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:28988/
   [junit4]   2> 1519378 T3949 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:29040/
   [junit4]   2> 1519378 T3949 oasc.SocketProxy.close WARN Closing 6 connections to: http://127.0.0.1:29078/
   [junit4]   2> 1519379 T3949 oasc.SocketProxy.close WARN Closing 9 connections to: http://127.0.0.1:28973/
   [junit4]   2> 1519379 T3949 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 1519379 T3949 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:28988/
   [junit4]   2> 1519379 T3949 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:29040/
   [junit4]   2> 1519379 T3949 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:29078/
   [junit4]   2> 1519380 T3949 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:28973/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=20B7E01448DCCF36 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=pl -Dtests.timezone=Pacific/Enderbury -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR    119s J2 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28973/repfacttest_c8n_1x3_shard1_replica3
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([20B7E01448DCCF36:A8E3DFCEE620A2CE]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28973/repfacttest_c8n_1x3_shard1_replica3
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 48 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
   [junit4]    > 	... 53 more
   [junit4]   2> 1519402 T3949 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest 20B7E01448DCCF36-001
   [junit4]   2> 118734 T3948 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {range_facet_l=FST50, timestamp=FST50, text=BlockTreeOrds(blocksize=128), id=PostingsFormat(name=LuceneFixedGap), multiDefault=FST50, a_t=FST50, intDefault=PostingsFormat(name=LuceneVarGapFixedInterval), _version_=PostingsFormat(name=LuceneVarGapFixedInterval)}, docValues:{timestamp=DocValuesFormat(name=Memory)}, sim=DefaultSimilarity, locale=pl, timezone=Pacific/Enderbury
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=227164312,total=367001600
   [junit4]   2> NOTE: All tests run in this JVM: [TestUtils, TestSolrConfigHandlerCloud, TestDefaultStatsCache, LeaderElectionIntegrationTest, TestCloudSchemaless, TestCSVResponseWriter, TestQuerySenderNoQuery, DistributedQueueTest, ActionThrottleTest, TestCharFilters, BasicZkTest, TestExpandComponent, CloudExitableDirectoryReaderTest, ShardRoutingTest, TestOverriddenPrefixQueryForCustomFieldType, TestPostingsSolrHighlighter, SyncSliceTest, TestHighlightDedupGrouping, RequiredFieldsTest, HdfsRecoveryZkTest, TestElisionMultitermQuery, RankQueryTest, RequestLoggingTest, TestStressUserVersions, DeleteLastCustomShardedReplicaTest, SpellCheckCollatorTest, TestSolrJ, TestCloudPivotFacet, TestArbitraryIndexDir, TestConfig, TestSolrConfigHandlerConcurrent, TestSchemaNameResource, TestJsonFacets, TestDistributedSearch, TestFreeTextSuggestions, TestSolrQueryParserResource, ExitableDirectoryReaderTest, TestSerializedLuceneMatchVersion, TestFoldingMultitermQuery, DirectSolrConnectionTest, TestPseudoReturnFields, IndexSchemaTest, BadIndexSchemaTest, HardAutoCommitTest, FastVectorHighlighterTest, DistribCursorPagingTest, DistributedFacetPivotLongTailTest, ConvertedLegacyTest, TestSurroundQueryParser, ZkSolrClientTest, BufferStoreTest, TestDefaultSearchFieldResource, TestOmitPositions, SuggesterTest, TestCoreDiscovery, TestShortCircuitedRequests, InfoHandlerTest, TestDocumentBuilder, AnalysisErrorHandlingTest, FullSolrCloudDistribCmdsTest, ReplicationFactorTest]
   [junit4] Completed on J2 in 119.52s, 1 test, 1 error <<< FAILURES!

[...truncated 704 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:529: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-5.x-Java7/lucene/common-build.xml:976: There were test failures: 475 suites, 1892 tests, 3 errors, 62 ignored (22 assumptions)

Total time: 51 minutes 52 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-5.x-Java7 #2431
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 210190399 bytes
Compression is 0.0%
Took 41 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure